From e639f12f96960d4f1d27b5c6097304b72692a8c3 Mon Sep 17 00:00:00 2001 From: Tom Villette Date: Thu, 12 Oct 2023 18:42:17 +0200 Subject: [PATCH] transition to postgres --- .env.template | 4 +- docker-compose.dev.yml | 41 +++++------ eveal/database.py | 9 ++- eveal/esi.py | 7 +- import_sde.py | 151 ++++++++++++++++++++++++++--------------- requirements.txt | 1 + 6 files changed, 128 insertions(+), 85 deletions(-) diff --git a/.env.template b/.env.template index 2aeaa25..c5f00e2 100644 --- a/.env.template +++ b/.env.template @@ -5,12 +5,10 @@ ESI_USER_AGENT= REDIS_URL= REDIS_PORT= - -REDIS_USER= REDIS_PASSWORD= -REDIS_SSL= SQLITE_DB_PATH= +POSTGRES_HOST= POSTGRES_PASSWORD= POSTGRES_USER= POSTGRES_DB= diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 337c274..db7df2b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,7 @@ version: '3' services: eveal: - image: eveal:latest + image: mabras:local build: context: . dockerfile: Dockerfile @@ -11,13 +11,12 @@ services: - 8000:8000 volumes: - ./eveal:/app/eveal - - ./eveal.db:/app/eveal.db command: ["uvicorn", "eveal.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] depends_on: redis: condition: service_healthy -# db: -# condition: service_healthy + db: + condition: service_healthy # elasticsearch: # condition: service_healthy @@ -29,22 +28,21 @@ services: test: redis-cli ping interval: 3s -# db: -# image: postgres:13-alpine -# ports: -# - 5432:5432 -# volumes: -# - ./dump_sde.sql:/docker-entrypoint-initdb.d/init_sde.sql -# - mabras_dbdata:/var/lib/postgresql/data -# environment: -# - POSTGRES_PASSWORD -# - POSTGRES_USER -# - POSTGRES_DB -# healthcheck: -# test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} -# interval: 10s -# timeout: 3s -# retries: 3 + db: + image: postgres:13-alpine + ports: + - 5432:5432 + volumes: + - mabras_dbdata:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD + - POSTGRES_USER + - POSTGRES_DB + healthcheck: + test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} + interval: 10s + timeout: 3s + retries: 3 # elasticsearch: # image: elasticsearch:latest @@ -60,3 +58,6 @@ services: # interval: 10s # timeout: 3s # retries: 3 + +volumes: + mabras_dbdata: diff --git a/eveal/database.py b/eveal/database.py index 96e955e..9c9287e 100644 --- a/eveal/database.py +++ b/eveal/database.py @@ -1,10 +1,13 @@ import os from sqlmodel import create_engine, Session -from eveal import models_sde -sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db") -engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=True, future=True, connect_args={"check_same_thread": False}) +if os.getenv("POSTGRES_HOST"): + engine = create_engine(f"postgresql://{os.getenv('POSTGRES_USER')}:{os.getenv('POSTGRES_PASSWORD')}@{os.getenv('POSTGRES_HOST')}/{os.getenv('POSTGRES_DB')}", + echo=False, future=True) +else: + sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db") + engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=False, future=True, connect_args={"check_same_thread": False}) def get_session(): db = Session(engine) diff --git a/eveal/esi.py b/eveal/esi.py index aeebe2e..d186d94 100644 --- a/eveal/esi.py +++ b/eveal/esi.py @@ -7,8 +7,8 @@ from esy.auth import ESIAuthenticator class ESICache(object): - def __init__(self, redis_url: str, redis_port: int, db: str): - self._r = redis.Redis(host=redis_url, port=redis_port, db=db) + def __init__(self, **kwargs): + self._r = redis.Redis(**kwargs) # self._r = redis.StrictRedis(host=redis_url, port=redis_port, db=db) def get(self, key): @@ -26,7 +26,8 @@ class ESICache(object): esi_client_id = os.getenv('ESI_CLIENT_ID') esi_secret_key = os.getenv('ESI_SECRET_KEY') -esi_cache = ESICache(redis_url=os.getenv("REDIS_URL"), redis_port=int(os.getenv("REDIS_PORT")), db="0") +esi_cache = ESICache(host=os.getenv("REDIS_URL"), port=int(os.getenv("REDIS_PORT")), db="0", + password=os.getenv("REDIS_PASSWD")) esi_client = ESIClient.get_client(user_agent=os.getenv('ESI_USER_AGENT'), cache=esi_cache) esi_auth = ESIAuthenticator() diff --git a/import_sde.py b/import_sde.py index f3d7965..72226cb 100644 --- a/import_sde.py +++ b/import_sde.py @@ -1,13 +1,8 @@ import yaml -from eveal.database import engine, sqlite_file_name -from sqlmodel import Session, SQLModel +from eveal.database import engine +from sqlmodel import Session, SQLModel, select from eveal import models_sde -import os -try: - os.remove(sqlite_file_name) -except: - pass SQLModel.metadata.create_all(engine) @@ -15,99 +10,143 @@ print("Importing SDE data...") print("Importing icons...") with open("static_eve/sde/fsd/iconIDs.yaml", "r", encoding="utf-8") as f: icons = yaml.safe_load(f) - +new_icons = total_icons = 0 with Session(engine) as db: for id, icon in icons.items(): - db.add(models_sde.SDEIcon(id=id, **icon)) + db_icon = db.get(models_sde.SDEIcon, id) + if not db_icon: + db_icon = models_sde.SDEIcon(id=id) + db.add(db_icon) + new_icons += 1 + db_icon.iconFile = icon['iconFile'] + db_icon.description = icon['description'] if 'description' in icon else None + total_icons += 1 db.commit() +print(f"Imported {new_icons} new icons / {total_icons} total icons.") print("Importing categories...") with open("static_eve/sde/fsd/categoryIDs.yaml", "r", encoding="utf-8") as f: categories = yaml.safe_load(f) - +new_categories = total_categories = 0 with Session(engine) as db: for id, category in categories.items(): - if category["published"] == False: - continue - db.add(models_sde.SDECategory(id=id, - icon_id=category['iconID'] if 'iconID' in category else None, - name=category['name']['en'], - published=category['published'])) + # if category["published"] == False: + # continue + db_category = db.get(models_sde.SDECategory, id) + if not db_category: + db_category = models_sde.SDECategory(id=id) + db.add(db_category) + new_categories += 1 + db_category.name = category['name']['en'] + db_category.published = category['published'] + db_category.icon_id = category['iconID'] if 'iconID' in category else None + total_categories += 1 db.commit() +print(f"Imported {new_categories} new categories / {total_categories} total categories.") print("Importing groups...") with open("static_eve/sde/fsd/groupIDs.yaml", "r", encoding="utf-8") as f: groups = yaml.safe_load(f) - +new_groups = total_groups = 0 with Session(engine) as db: for id, group in groups.items(): - if group["published"] == False: - continue - db.add(models_sde.SDEGroup(id=id, - anchorable=group['anchorable'], - anchored=group['anchored'], - category_id=group['categoryID'], - fittableNonSingletion=group['fittableNonSingleton'], - icon_id=group['iconID'] if 'iconID' in group else None, - name=group['name']['en'], - published=group['published'], - useBasePrice=group['useBasePrice'] - )) + # if group["published"] == False: + # continue + db_group = db.get(models_sde.SDEGroup, id) + if not db_group: + db_group = models_sde.SDEGroup(id=id) + db.add(db_group) + new_groups += 1 + db_group.anchorable = group['anchorable'] + db_group.anchored = group['anchored'] + db_group.category_id = group['categoryID'] + db_group.fittableNonSingletion = group['fittableNonSingleton'] + db_group.icon_id = group['iconID'] if 'iconID' in group else None + db_group.name = group['name']['en'] + db_group.published = group['published'] + db_group.useBasePrice = group['useBasePrice'] + total_groups += 1 db.commit() +print(f"Imported {new_groups} new groups / {total_groups} total groups.") print("Importing marketgroups...") with open("static_eve/sde/fsd/marketGroups.yaml", "r", encoding="utf-8") as f: marketgroups = yaml.safe_load(f) - +new_marketgroups = total_marketgroups = 0 with Session(engine) as db: for id, marketgroup in marketgroups.items(): - db.add(models_sde.SDEMarketGroup(id=id, - description=marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None, - hasTypes=marketgroup['hasTypes'], - icon_id=marketgroup['iconID'] if 'iconID' in marketgroup else None, - name=marketgroup['nameID']['en'], - parent_marketgroup_id=marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None, - )) + db_marketgroups = db.get(models_sde.SDEMarketGroup, id) + if not db_marketgroups: + db_marketgroups = models_sde.SDEMarketGroup(id=id) + db.add(db_marketgroups) + new_marketgroups += 1 + db_marketgroups.description = marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None + db_marketgroups.hasTypes = marketgroup['hasTypes'] + db_marketgroups.icon_id = marketgroup['iconID'] if 'iconID' in marketgroup else None + db_marketgroups.name = marketgroup['nameID']['en'] + # db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None + total_marketgroups += 1 db.commit() +print(f"Imported {new_marketgroups} marketgroups / {total_marketgroups} total marketgroups.") +print("Setting up marketgroup Parents...") +total_marketgroup_links = 0 +with Session(engine) as db: + for id, marketgroup in marketgroups.items(): + db_marketgroups = db.get(models_sde.SDEMarketGroup, id) + db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None + total_marketgroup_links += 1 + db.commit() +print(f"Updated {total_marketgroup_links} marketgroup Parents") print("Importing types...") with open("static_eve/sde/fsd/typeIDs.yaml", "r", encoding="utf-8") as f: types = yaml.safe_load(f) - +new_types = total_types = 0 with Session(engine) as db: for id, type in types.items(): - if type["published"] == False: - continue - db.add(models_sde.SDEType(id=id, - group_id=type['groupID'], - marketgroup_id=type['marketGroupID'] if 'marketGroupID' in type else None, - name=type['name']['en'], - published=type['published'], - basePrice=type['basePrice'] if 'basePrice' in type else None, - description=type['description']['en'] if 'description' in type else None, - icon_id=type['iconID'] if 'iconID' in type else None, - portionSize=type['portionSize'], - volume=type['volume'] if 'volume' in type else None, - )) + # if type["published"] == False: + # continue + db_type = db.get(models_sde.SDEType, id) + if not db_type: + db_type = models_sde.SDEType(id=id) + db.add(db_type) + new_types += 1 + db_type.group_id = type['groupID'] + db_type.marketgroup_id = type['marketGroupID'] if 'marketGroupID' in type else None + db_type.name = type['name']['en'] + db_type.published = type['published'] + db_type.basePrice = type['basePrice'] if 'basePrice' in type else None + db_type.description = type['description']['en'] if 'description' in type else None + db_type.icon_id = type['iconID'] if 'iconID' in type else None + db_type.portionSize = type['portionSize'] + db_type.volume = type['volume'] if 'volume' in type else None + total_types += 1 db.commit() +print(f"Imported {new_types} types / {total_types} total types.") print("Importing materials...") with open("static_eve/sde/fsd/typeMaterials.yaml", "r", encoding="utf-8") as f: materials = yaml.safe_load(f) - +new_typemat = total_typemat = 0 with Session(engine) as db: for id, material in materials.items(): for mat in material['materials']: - db.add(models_sde.SDETypeMaterial(type_id=id, - material_id=mat['materialTypeID'], - quantity=mat['quantity'] - )) + db_typemat = db.exec(select(models_sde.SDETypeMaterial).where(models_sde.SDETypeMaterial.type_id == id, models_sde.SDETypeMaterial.material_type_id == mat['materialTypeID'])).one_or_none() + if not db_typemat: + db_typemat = models_sde.SDETypeMaterial() + db.add(db_typemat) + new_typemat += 1 + db_typemat.type_id = id + db_typemat.material_type_id = mat['materialTypeID'] + db_typemat.quantity = mat['quantity'] + total_typemat += 1 db.commit() +print(f"Imported {new_typemat} materials / {total_typemat} total materials.") print("DONE!") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d0e1083..8c0c8cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ uvicorn[standard] sqlmodel esy redis +psycopg2-binary