From c3002d201fadca0d9538a44179d67b3adc1a3537 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Thu, 26 Oct 2023 20:58:46 +0200 Subject: [PATCH] new import_sde --- api/management/__init__.py | 0 api/management/commands/__init__.py | 0 api/management/commands/import_sde.py | 140 ++++++++++++++++++++++++++ api/migrations/0001_initial.py | 10 +- api/models_sde.py | 8 +- docker-compose.dev.yml | 3 +- requirements.txt | 2 + 7 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 api/management/__init__.py create mode 100644 api/management/commands/__init__.py create mode 100644 api/management/commands/import_sde.py diff --git a/api/management/__init__.py b/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/__init__.py b/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/import_sde.py b/api/management/commands/import_sde.py new file mode 100644 index 0000000..1fecf3b --- /dev/null +++ b/api/management/commands/import_sde.py @@ -0,0 +1,140 @@ +from django.core.management.base import BaseCommand, CommandError +from api.models import * +import yaml + +class Command(BaseCommand): + help = 'Import SDE data from YAML files' + + def add_arguments(self, parser): + parser.add_argument('path', type=str, help='Path to the SDE YAML files') + + def handle(self, *args, **options): + # Import the SDE data + self.stdout.write(self.style.SUCCESS('Importing SDE data')) + self.import_sde_data(path=options['path']) + self.stdout.write(self.style.SUCCESS('Successfully imported SDE data')) + + def import_sde_data(self, path): + self.import_icons(path + "/iconIDs.yaml") + self.import_categories(path + "/categoryIDs.yaml") + self.import_groups(path + "/groupIDs.yaml") + self.import_marketgroups(path + "/marketGroups.yaml") + self.import_metagroups(path + "/metaGroups.yaml") + self.import_types(path + "/typeIDs.yaml") + self.import_type_materials(path + "/typeMaterials.yaml") + + def import_icons(self, path): + with open(path) as file: + icons = yaml.load(file, Loader=yaml.FullLoader) + for id, icon in icons.items(): + SDEIcon.objects.update_or_create( + id=id, + defaults={ + 'iconFile': icon['iconFile'], + 'description': icon['description'] if "description" in icon else "" + } + ) + self.stdout.write(self.style.SUCCESS('Icons imported')) + + def import_categories(self, path): + with open(path) as file: + categories = yaml.load(file, Loader=yaml.FullLoader) + for id, category in categories.items(): + SDECategory.objects.update_or_create( + id=id, + defaults={ + 'icon': SDEIcon.objects.get(id=category['iconID']) if "iconID" in category else None, + 'name': category['name']['en'], + 'published': category['published'] + } + ) + self.stdout.write(self.style.SUCCESS('Categories imported')) + + def import_groups(self, path): + with open(path) as file: + groups = yaml.load(file, Loader=yaml.FullLoader) + for id, group in groups.items(): + SDEGroup.objects.update_or_create( + id=id, + defaults={ + 'category': SDECategory.objects.get(id=group['categoryID']), + 'name': group['name'], + 'published': group['published'], + 'useBasePrice': group['useBasePrice'], + 'fittableNonSingleton': group['fittableNonSingleton'], + 'anchored': group['anchored'], + 'anchorable': group['anchorable'], + 'icon': SDEIcon.objects.get(id=group['iconID']) if "iconID" in group else None + } + ) + self.stdout.write(self.style.SUCCESS('Groups imported')) + + def import_marketgroups(self, path): + with open(path) as file: + marketgroups = yaml.load(file, Loader=yaml.FullLoader) + for id, marketgroup in marketgroups.items(): + SDEMarektGroup.objects.update_or_create( + id=id, + defaults={ + 'icon': SDEIcon.objects.get(id=marketgroup['iconID']) if "iconID" in marketgroup else None, + 'name': marketgroup['nameID']['en'], + 'description': marketgroup['descriptionID']['en'] if "descriptionID" in marketgroup else "", + 'hasTypes': marketgroup['hasTypes'], + } + ) + self.stdout.write(self.style.SUCCESS('Marketgroups imported')) + for id, marketgroup in marketgroups.items(): + if "parentGroupID" in marketgroup: + sde_mg = SDEMarektGroup.objects.get(id=id) + sde_mg.parent_marketgroup = SDEMarektGroup.objects.get(id=marketgroup['parentGroupID']) + sde_mg.save() + self.stdout.write(self.style.SUCCESS('Marketgroups linked')) + + def import_metagroups(self, path): + with open(path) as file: + metagroups = yaml.load(file, Loader=yaml.FullLoader) + for id, metagroup in metagroups.items(): + SDEMetaGroup.objects.update_or_create( + id=id, + defaults={ + 'icon': SDEIcon.objects.get(id=metagroup['iconID']) if "iconID" in metagroup else None, + 'name': metagroup['nameID']['en'], + 'iconSuffix': metagroup['iconSuffix'] if "iconSuffix" in metagroup else "", + } + ) + self.stdout.write(self.style.SUCCESS('Metagroups imported')) + + def import_types(self, path): + with open(path) as file: + types = yaml.load(file, Loader=yaml.FullLoader) + for id, type in types.items(): + SDEType.objects.update_or_create( + id=id, + defaults={ + 'group': SDEGroup.objects.get(id=type['groupID']), + 'marketgroup': SDEMarektGroup.objects.get(id=type['marketGroupID']) if "marketGroupID" in type else None, + 'metagroup': SDEMetaGroup.objects.get(id=type['metaGroupID']) if "metaGroupID" in type else None, + 'name': type['name']['en'], + 'description': type['description']['en'] if "description" in type else "", + 'published': type['published'], + 'basePrice': type['basePrice'] if "basePrice" in type else 0, + 'icon': SDEIcon.objects.get(id=type['iconID']) if "iconID" in type else None, + 'volume': type['volume'] if "volume" in type else 0, + 'portionSize': type['portionSize'], + } + ) + self.stdout.write(self.style.SUCCESS('Types imported')) + + def import_type_materials(self, path): + with open(path) as file: + type_materials = yaml.load(file, Loader=yaml.FullLoader) + for id, type_material in type_materials.items(): + for material in type_material['materials']: + SDETypeMaterial.objects.update_or_create( + type=SDEType.objects.get(id=id), + material_type=SDEType.objects.get(id=material['materialTypeID']), + defaults={ + 'quantity': material['quantity'], + } + ) + self.stdout.write(self.style.SUCCESS('Materials imported')) diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py index 77f2d3a..4297bfd 100644 --- a/api/migrations/0001_initial.py +++ b/api/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.6 on 2023-10-26 15:51 +# Generated by Django 4.2.6 on 2023-10-26 18:34 from django.db import migrations, models import django.db.models.deletion @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('name', models.CharField()), ('published', models.BooleanField()), ('useBasePrice', models.BooleanField()), - ('fittableNonSingletion', models.BooleanField()), + ('fittableNonSingleton', models.BooleanField()), ('anchored', models.BooleanField()), ('anchorable', models.BooleanField()), ('category', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='api.sdecategory')), @@ -73,13 +73,13 @@ class Migration(migrations.Migration): ('portionSize', models.IntegerField()), ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdegroup')), ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdeicon')), - ('marketgroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdemarektgroup')), + ('marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdemarektgroup')), ], ), migrations.CreateModel( name='SDETypeMaterial', fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('quantity', models.IntegerField()), ('material_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials_of', to='api.sdetype')), ('type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials', to='api.sdetype')), @@ -93,7 +93,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='sdetype', name='metagroup', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdemetagroup'), + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdemetagroup'), ), migrations.AddField( model_name='sdegroup', diff --git a/api/models_sde.py b/api/models_sde.py index 8b3ff62..2b38ec7 100644 --- a/api/models_sde.py +++ b/api/models_sde.py @@ -20,7 +20,7 @@ class SDEGroup(models.Model): name = models.CharField() published = models.BooleanField() useBasePrice = models.BooleanField() - fittableNonSingletion = models.BooleanField() + fittableNonSingleton = models.BooleanField() anchored = models.BooleanField() anchorable = models.BooleanField() icon = models.ForeignKey(SDEIcon, related_name="groups", null=True, on_delete=models.SET_NULL) @@ -45,8 +45,8 @@ class SDEMetaGroup(models.Model): class SDEType(models.Model): id = models.IntegerField(primary_key=True) group = models.ForeignKey(SDEGroup, related_name="types", on_delete=models.CASCADE) - marketgroup = models.ForeignKey(SDEMarektGroup, related_name="types", on_delete=models.CASCADE) - metagroup = models.ForeignKey(SDEMetaGroup, related_name="types", on_delete=models.CASCADE) + marketgroup = models.ForeignKey(SDEMarektGroup, related_name="types", on_delete=models.SET_NULL, null=True) + metagroup = models.ForeignKey(SDEMetaGroup, related_name="types", on_delete=models.SET_NULL, null=True) name = models.CharField() description = models.CharField() published = models.BooleanField() @@ -58,9 +58,7 @@ class SDEType(models.Model): class SDETypeMaterial(models.Model): - id = models.IntegerField(primary_key=True) type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials") material_type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials_of") quantity = models.IntegerField() - diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 46800e9..90c594f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -12,7 +12,7 @@ services: - ./mabras:/app/mabras - ./api:/app/api - ./manage.py:/app/manage.py - command: "python manage.py makemigrations;python manage.py migrate" + command: sh -c "python manage.py makemigrations && python manage.py migrate" depends_on: db: condition: service_healthy @@ -28,6 +28,7 @@ services: - ./mabras:/app/mabras - ./api:/app/api - ./manage.py:/app/manage.py + - ./static_eve:/app/static_eve command: ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000", "--reload"] depends_on: redis: diff --git a/requirements.txt b/requirements.txt index 32efb56..b87e5c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ django-filter markdown uvicorn[standard] psycopg[binary] +redis +esy