transition to postgres

import_sde.py

@@ -1,13 +1,8 @@
 import yaml
-from eveal.database import engine, sqlite_file_name
-from sqlmodel import Session, SQLModel
+from eveal.database import engine
+from sqlmodel import Session, SQLModel, select
 from eveal import models_sde
 
-import os
-try:
-    os.remove(sqlite_file_name)
-except:
-    pass
 SQLModel.metadata.create_all(engine)
 
 
@@ -15,99 +10,143 @@ print("Importing SDE data...")
 print("Importing icons...")
 with open("static_eve/sde/fsd/iconIDs.yaml", "r", encoding="utf-8") as f:
     icons = yaml.safe_load(f)
-
+new_icons = total_icons = 0
 with Session(engine) as db:
     for id, icon in icons.items():
-        db.add(models_sde.SDEIcon(id=id, **icon))
+        db_icon = db.get(models_sde.SDEIcon, id)
+        if not db_icon:
+            db_icon = models_sde.SDEIcon(id=id)
+            db.add(db_icon)
+            new_icons += 1
+        db_icon.iconFile = icon['iconFile']
+        db_icon.description = icon['description'] if 'description' in icon else None
+        total_icons += 1
     db.commit()
+print(f"Imported {new_icons} new icons / {total_icons} total icons.")
 
 
 print("Importing categories...")
 with open("static_eve/sde/fsd/categoryIDs.yaml", "r", encoding="utf-8") as f:
     categories = yaml.safe_load(f)
-
+new_categories = total_categories = 0
 with Session(engine) as db:
     for id, category in categories.items():
-        if category["published"] == False:
-            continue
-        db.add(models_sde.SDECategory(id=id,
-                                      icon_id=category['iconID'] if 'iconID' in category else None,
-                                      name=category['name']['en'],
-                                      published=category['published']))
+        # if category["published"] == False:
+        #     continue
+        db_category = db.get(models_sde.SDECategory, id)
+        if not db_category:
+            db_category = models_sde.SDECategory(id=id)
+            db.add(db_category)
+            new_categories += 1
+        db_category.name = category['name']['en']
+        db_category.published = category['published']
+        db_category.icon_id = category['iconID'] if 'iconID' in category else None
+        total_categories += 1
     db.commit()
+print(f"Imported {new_categories} new categories / {total_categories} total categories.")
 
 
 print("Importing groups...")
 with open("static_eve/sde/fsd/groupIDs.yaml", "r", encoding="utf-8") as f:
     groups = yaml.safe_load(f)
-
+new_groups = total_groups = 0
 with Session(engine) as db:
     for id, group in groups.items():
-        if group["published"] == False:
-            continue
-        db.add(models_sde.SDEGroup(id=id,
-                                   anchorable=group['anchorable'],
-                                   anchored=group['anchored'],
-                                   category_id=group['categoryID'],
-                                   fittableNonSingletion=group['fittableNonSingleton'],
-                                   icon_id=group['iconID'] if 'iconID' in group else None,
-                                   name=group['name']['en'],
-                                   published=group['published'],
-                                   useBasePrice=group['useBasePrice']
-                                   ))
+        # if group["published"] == False:
+        #     continue
+        db_group = db.get(models_sde.SDEGroup, id)
+        if not db_group:
+            db_group = models_sde.SDEGroup(id=id)
+            db.add(db_group)
+            new_groups += 1
+        db_group.anchorable = group['anchorable']
+        db_group.anchored = group['anchored']
+        db_group.category_id = group['categoryID']
+        db_group.fittableNonSingletion = group['fittableNonSingleton']
+        db_group.icon_id = group['iconID'] if 'iconID' in group else None
+        db_group.name = group['name']['en']
+        db_group.published = group['published']
+        db_group.useBasePrice = group['useBasePrice']
+        total_groups += 1
     db.commit()
+print(f"Imported {new_groups} new groups / {total_groups} total groups.")
 
 
 print("Importing marketgroups...")
 with open("static_eve/sde/fsd/marketGroups.yaml", "r", encoding="utf-8") as f:
     marketgroups = yaml.safe_load(f)
-
+new_marketgroups = total_marketgroups = 0
 with Session(engine) as db:
     for id, marketgroup in marketgroups.items():
-        db.add(models_sde.SDEMarketGroup(id=id,
-                                         description=marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None,
-                                         hasTypes=marketgroup['hasTypes'],
-                                         icon_id=marketgroup['iconID'] if 'iconID' in marketgroup else None,
-                                         name=marketgroup['nameID']['en'],
-                                         parent_marketgroup_id=marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None,
-                                         ))
+        db_marketgroups = db.get(models_sde.SDEMarketGroup, id)
+        if not db_marketgroups:
+            db_marketgroups = models_sde.SDEMarketGroup(id=id)
+            db.add(db_marketgroups)
+            new_marketgroups += 1
+        db_marketgroups.description = marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None
+        db_marketgroups.hasTypes = marketgroup['hasTypes']
+        db_marketgroups.icon_id = marketgroup['iconID'] if 'iconID' in marketgroup else None
+        db_marketgroups.name = marketgroup['nameID']['en']
+        # db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None
+        total_marketgroups += 1
     db.commit()
+print(f"Imported {new_marketgroups} marketgroups / {total_marketgroups} total marketgroups.")
+print("Setting up marketgroup Parents...")
+total_marketgroup_links = 0
+with Session(engine) as db:
+    for id, marketgroup in marketgroups.items():
+        db_marketgroups = db.get(models_sde.SDEMarketGroup, id)
+        db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None
+        total_marketgroup_links += 1
+    db.commit()
+print(f"Updated {total_marketgroup_links} marketgroup Parents")
 
 
 print("Importing types...")
 with open("static_eve/sde/fsd/typeIDs.yaml", "r", encoding="utf-8") as f:
     types = yaml.safe_load(f)
-
+new_types = total_types = 0
 with Session(engine) as db:
     for id, type in types.items():
-        if type["published"] == False:
-            continue
-        db.add(models_sde.SDEType(id=id,
-                                  group_id=type['groupID'],
-                                  marketgroup_id=type['marketGroupID'] if 'marketGroupID' in type else None,
-                                  name=type['name']['en'],
-                                  published=type['published'],
-                                  basePrice=type['basePrice'] if 'basePrice' in type else None,
-                                  description=type['description']['en'] if 'description' in type else None,
-                                  icon_id=type['iconID'] if 'iconID' in type else None,
-                                  portionSize=type['portionSize'],
-                                  volume=type['volume'] if 'volume' in type else None,
-                                  ))
+        # if type["published"] == False:
+        #     continue
+        db_type = db.get(models_sde.SDEType, id)
+        if not db_type:
+            db_type = models_sde.SDEType(id=id)
+            db.add(db_type)
+            new_types += 1
+        db_type.group_id = type['groupID']
+        db_type.marketgroup_id = type['marketGroupID'] if 'marketGroupID' in type else None
+        db_type.name = type['name']['en']
+        db_type.published = type['published']
+        db_type.basePrice = type['basePrice'] if 'basePrice' in type else None
+        db_type.description = type['description']['en'] if 'description' in type else None
+        db_type.icon_id = type['iconID'] if 'iconID' in type else None
+        db_type.portionSize = type['portionSize']
+        db_type.volume = type['volume'] if 'volume' in type else None
+        total_types += 1
     db.commit()
+print(f"Imported {new_types} types / {total_types} total types.")
 
 
 print("Importing materials...")
 with open("static_eve/sde/fsd/typeMaterials.yaml", "r", encoding="utf-8") as f:
     materials = yaml.safe_load(f)
-
+new_typemat = total_typemat = 0
 with Session(engine) as db:
     for id, material in materials.items():
         for mat in material['materials']:
-            db.add(models_sde.SDETypeMaterial(type_id=id,
-                                              material_id=mat['materialTypeID'],
-                                              quantity=mat['quantity']
-                                              ))
+            db_typemat = db.exec(select(models_sde.SDETypeMaterial).where(models_sde.SDETypeMaterial.type_id == id, models_sde.SDETypeMaterial.material_type_id == mat['materialTypeID'])).one_or_none()
+            if not db_typemat:
+                db_typemat = models_sde.SDETypeMaterial()
+                db.add(db_typemat)
+                new_typemat += 1
+            db_typemat.type_id = id
+            db_typemat.material_type_id = mat['materialTypeID']
+            db_typemat.quantity = mat['quantity']
+            total_typemat += 1
     db.commit()
+print(f"Imported {new_typemat} materials / {total_typemat} total materials.")
 
 
 print("DONE!")