eveal/marbas
2
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

transition to postgres

Browse Source
This commit is contained in:
Tom Villette
2023-10-12 18:42:17 +02:00
parent 4ce0b80196
commit e639f12f96
6 changed files with 128 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.env.template
View File

@@ -5,12 +5,10 @@ ESI_USER_AGENT=
REDIS_URL= REDIS_URL=
REDIS_PORT= REDIS_PORT=
REDIS_USER=
REDIS_PASSWORD= REDIS_PASSWORD=
REDIS_SSL=
SQLITE_DB_PATH= SQLITE_DB_PATH=
POSTGRES_HOST=
POSTGRES_PASSWORD= POSTGRES_PASSWORD=
POSTGRES_USER= POSTGRES_USER=
POSTGRES_DB= POSTGRES_DB=

41
docker-compose.dev.yml
View File

@@ -1,7 +1,7 @@
version: '3' version: '3'
services: services:
eveal: eveal:
image: eveal:latest image: mabras:local
build: build:
context: . context: .
dockerfile: Dockerfile dockerfile: Dockerfile
@@ -11,13 +11,12 @@ services:
- 8000:8000 - 8000:8000
volumes: volumes:
- ./eveal:/app/eveal - ./eveal:/app/eveal
- ./eveal.db:/app/eveal.db
command: ["uvicorn", "eveal.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] command: ["uvicorn", "eveal.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
depends_on: depends_on:
redis: redis:
condition: service_healthy condition: service_healthy
# db: db:
# condition: service_healthy condition: service_healthy
# elasticsearch: # elasticsearch:
# condition: service_healthy # condition: service_healthy
@@ -29,22 +28,21 @@ services:
test: redis-cli ping test: redis-cli ping
interval: 3s interval: 3s
# db: db:
# image: postgres:13-alpine image: postgres:13-alpine
# ports: ports:
# - 5432:5432 - 5432:5432
# volumes: volumes:
# - ./dump_sde.sql:/docker-entrypoint-initdb.d/init_sde.sql - mabras_dbdata:/var/lib/postgresql/data
# - mabras_dbdata:/var/lib/postgresql/data environment:
# environment: - POSTGRES_PASSWORD
# - POSTGRES_PASSWORD - POSTGRES_USER
# - POSTGRES_USER - POSTGRES_DB
# - POSTGRES_DB healthcheck:
# healthcheck: test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}
# test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} interval: 10s
# interval: 10s timeout: 3s
# timeout: 3s retries: 3
# retries: 3
# elasticsearch: # elasticsearch:
# image: elasticsearch:latest # image: elasticsearch:latest
@@ -60,3 +58,6 @@ services:
# interval: 10s # interval: 10s
# timeout: 3s # timeout: 3s
# retries: 3 # retries: 3
volumes:
mabras_dbdata:

7
eveal/database.py
View File

@@ -1,10 +1,13 @@
import os import os
from sqlmodel import create_engine, Session from sqlmodel import create_engine, Session
from eveal import models_sde
if os.getenv("POSTGRES_HOST"):
engine = create_engine(f"postgresql://{os.getenv('POSTGRES_USER')}:{os.getenv('POSTGRES_PASSWORD')}@{os.getenv('POSTGRES_HOST')}/{os.getenv('POSTGRES_DB')}",
echo=False, future=True)
else:
sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db") sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db")
engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=True, future=True, connect_args={"check_same_thread": False}) engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=False, future=True, connect_args={"check_same_thread": False})
def get_session(): def get_session():
db = Session(engine) db = Session(engine)

7
eveal/esi.py
View File

@@ -7,8 +7,8 @@ from esy.auth import ESIAuthenticator
class ESICache(object): class ESICache(object):
def __init__(self, redis_url: str, redis_port: int, db: str): def __init__(self, **kwargs):
self._r = redis.Redis(host=redis_url, port=redis_port, db=db) self._r = redis.Redis(**kwargs)
# self._r = redis.StrictRedis(host=redis_url, port=redis_port, db=db) # self._r = redis.StrictRedis(host=redis_url, port=redis_port, db=db)
def get(self, key): def get(self, key):
@@ -26,7 +26,8 @@ class ESICache(object):
esi_client_id = os.getenv('ESI_CLIENT_ID') esi_client_id = os.getenv('ESI_CLIENT_ID')
esi_secret_key = os.getenv('ESI_SECRET_KEY') esi_secret_key = os.getenv('ESI_SECRET_KEY')
esi_cache = ESICache(redis_url=os.getenv("REDIS_URL"), redis_port=int(os.getenv("REDIS_PORT")), db="0") esi_cache = ESICache(host=os.getenv("REDIS_URL"), port=int(os.getenv("REDIS_PORT")), db="0",
password=os.getenv("REDIS_PASSWD"))
esi_client = ESIClient.get_client(user_agent=os.getenv('ESI_USER_AGENT'), cache=esi_cache) esi_client = ESIClient.get_client(user_agent=os.getenv('ESI_USER_AGENT'), cache=esi_cache)
esi_auth = ESIAuthenticator() esi_auth = ESIAuthenticator()

151
import_sde.py
View File

@@ -1,13 +1,8 @@
import yaml import yaml
from eveal.database import engine, sqlite_file_name from eveal.database import engine
from sqlmodel import Session, SQLModel from sqlmodel import Session, SQLModel, select
from eveal import models_sde from eveal import models_sde
import os
try:
os.remove(sqlite_file_name)
except:
pass
SQLModel.metadata.create_all(engine) SQLModel.metadata.create_all(engine)
@@ -15,99 +10,143 @@ print("Importing SDE data...")
print("Importing icons...") print("Importing icons...")
with open("static_eve/sde/fsd/iconIDs.yaml", "r", encoding="utf-8") as f: with open("static_eve/sde/fsd/iconIDs.yaml", "r", encoding="utf-8") as f:
icons = yaml.safe_load(f) icons = yaml.safe_load(f)
new_icons = total_icons = 0
with Session(engine) as db: with Session(engine) as db:
for id, icon in icons.items(): for id, icon in icons.items():
db.add(models_sde.SDEIcon(id=id, **icon)) db_icon = db.get(models_sde.SDEIcon, id)
if not db_icon:
db_icon = models_sde.SDEIcon(id=id)
db.add(db_icon)
new_icons += 1
db_icon.iconFile = icon['iconFile']
db_icon.description = icon['description'] if 'description' in icon else None
total_icons += 1
db.commit() db.commit()
print(f"Imported {new_icons} new icons / {total_icons} total icons.")
print("Importing categories...") print("Importing categories...")
with open("static_eve/sde/fsd/categoryIDs.yaml", "r", encoding="utf-8") as f: with open("static_eve/sde/fsd/categoryIDs.yaml", "r", encoding="utf-8") as f:
categories = yaml.safe_load(f) categories = yaml.safe_load(f)
new_categories = total_categories = 0
with Session(engine) as db: with Session(engine) as db:
for id, category in categories.items(): for id, category in categories.items():
if category["published"] == False: # if category["published"] == False:
continue # continue
db.add(models_sde.SDECategory(id=id, db_category = db.get(models_sde.SDECategory, id)
icon_id=category['iconID'] if 'iconID' in category else None, if not db_category:
name=category['name']['en'], db_category = models_sde.SDECategory(id=id)
published=category['published'])) db.add(db_category)
new_categories += 1
db_category.name = category['name']['en']
db_category.published = category['published']
db_category.icon_id = category['iconID'] if 'iconID' in category else None
total_categories += 1
db.commit() db.commit()
print(f"Imported {new_categories} new categories / {total_categories} total categories.")
print("Importing groups...") print("Importing groups...")
with open("static_eve/sde/fsd/groupIDs.yaml", "r", encoding="utf-8") as f: with open("static_eve/sde/fsd/groupIDs.yaml", "r", encoding="utf-8") as f:
groups = yaml.safe_load(f) groups = yaml.safe_load(f)
new_groups = total_groups = 0
with Session(engine) as db: with Session(engine) as db:
for id, group in groups.items(): for id, group in groups.items():
if group["published"] == False: # if group["published"] == False:
continue # continue
db.add(models_sde.SDEGroup(id=id, db_group = db.get(models_sde.SDEGroup, id)
anchorable=group['anchorable'], if not db_group:
anchored=group['anchored'], db_group = models_sde.SDEGroup(id=id)
category_id=group['categoryID'], db.add(db_group)
fittableNonSingletion=group['fittableNonSingleton'], new_groups += 1
icon_id=group['iconID'] if 'iconID' in group else None, db_group.anchorable = group['anchorable']
name=group['name']['en'], db_group.anchored = group['anchored']
published=group['published'], db_group.category_id = group['categoryID']
useBasePrice=group['useBasePrice'] db_group.fittableNonSingletion = group['fittableNonSingleton']
)) db_group.icon_id = group['iconID'] if 'iconID' in group else None
db_group.name = group['name']['en']
db_group.published = group['published']
db_group.useBasePrice = group['useBasePrice']
total_groups += 1
db.commit() db.commit()
print(f"Imported {new_groups} new groups / {total_groups} total groups.")
print("Importing marketgroups...") print("Importing marketgroups...")
with open("static_eve/sde/fsd/marketGroups.yaml", "r", encoding="utf-8") as f: with open("static_eve/sde/fsd/marketGroups.yaml", "r", encoding="utf-8") as f:
marketgroups = yaml.safe_load(f) marketgroups = yaml.safe_load(f)
new_marketgroups = total_marketgroups = 0
with Session(engine) as db: with Session(engine) as db:
for id, marketgroup in marketgroups.items(): for id, marketgroup in marketgroups.items():
db.add(models_sde.SDEMarketGroup(id=id, db_marketgroups = db.get(models_sde.SDEMarketGroup, id)
description=marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None, if not db_marketgroups:
hasTypes=marketgroup['hasTypes'], db_marketgroups = models_sde.SDEMarketGroup(id=id)
icon_id=marketgroup['iconID'] if 'iconID' in marketgroup else None, db.add(db_marketgroups)
name=marketgroup['nameID']['en'], new_marketgroups += 1
parent_marketgroup_id=marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None, db_marketgroups.description = marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None
)) db_marketgroups.hasTypes = marketgroup['hasTypes']
db_marketgroups.icon_id = marketgroup['iconID'] if 'iconID' in marketgroup else None
db_marketgroups.name = marketgroup['nameID']['en']
# db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None
total_marketgroups += 1
db.commit() db.commit()
print(f"Imported {new_marketgroups} marketgroups / {total_marketgroups} total marketgroups.")
print("Setting up marketgroup Parents...")
total_marketgroup_links = 0
with Session(engine) as db:
for id, marketgroup in marketgroups.items():
db_marketgroups = db.get(models_sde.SDEMarketGroup, id)
db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None
total_marketgroup_links += 1
db.commit()
print(f"Updated {total_marketgroup_links} marketgroup Parents")
print("Importing types...") print("Importing types...")
with open("static_eve/sde/fsd/typeIDs.yaml", "r", encoding="utf-8") as f: with open("static_eve/sde/fsd/typeIDs.yaml", "r", encoding="utf-8") as f:
types = yaml.safe_load(f) types = yaml.safe_load(f)
new_types = total_types = 0
with Session(engine) as db: with Session(engine) as db:
for id, type in types.items(): for id, type in types.items():
if type["published"] == False: # if type["published"] == False:
continue # continue
db.add(models_sde.SDEType(id=id, db_type = db.get(models_sde.SDEType, id)
group_id=type['groupID'], if not db_type:
marketgroup_id=type['marketGroupID'] if 'marketGroupID' in type else None, db_type = models_sde.SDEType(id=id)
name=type['name']['en'], db.add(db_type)
published=type['published'], new_types += 1
basePrice=type['basePrice'] if 'basePrice' in type else None, db_type.group_id = type['groupID']
description=type['description']['en'] if 'description' in type else None, db_type.marketgroup_id = type['marketGroupID'] if 'marketGroupID' in type else None
icon_id=type['iconID'] if 'iconID' in type else None, db_type.name = type['name']['en']
portionSize=type['portionSize'], db_type.published = type['published']
volume=type['volume'] if 'volume' in type else None, db_type.basePrice = type['basePrice'] if 'basePrice' in type else None
)) db_type.description = type['description']['en'] if 'description' in type else None
db_type.icon_id = type['iconID'] if 'iconID' in type else None
db_type.portionSize = type['portionSize']
db_type.volume = type['volume'] if 'volume' in type else None
total_types += 1
db.commit() db.commit()
print(f"Imported {new_types} types / {total_types} total types.")
print("Importing materials...") print("Importing materials...")
with open("static_eve/sde/fsd/typeMaterials.yaml", "r", encoding="utf-8") as f: with open("static_eve/sde/fsd/typeMaterials.yaml", "r", encoding="utf-8") as f:
materials = yaml.safe_load(f) materials = yaml.safe_load(f)
new_typemat = total_typemat = 0
with Session(engine) as db: with Session(engine) as db:
for id, material in materials.items(): for id, material in materials.items():
for mat in material['materials']: for mat in material['materials']:
db.add(models_sde.SDETypeMaterial(type_id=id, db_typemat = db.exec(select(models_sde.SDETypeMaterial).where(models_sde.SDETypeMaterial.type_id == id, models_sde.SDETypeMaterial.material_type_id == mat['materialTypeID'])).one_or_none()
material_id=mat['materialTypeID'], if not db_typemat:
quantity=mat['quantity'] db_typemat = models_sde.SDETypeMaterial()
)) db.add(db_typemat)
new_typemat += 1
db_typemat.type_id = id
db_typemat.material_type_id = mat['materialTypeID']
db_typemat.quantity = mat['quantity']
total_typemat += 1
db.commit() db.commit()
print(f"Imported {new_typemat} materials / {total_typemat} total materials.")
print("DONE!") print("DONE!")

1
requirements.txt
View File

@@ -4,3 +4,4 @@ uvicorn[standard]
sqlmodel sqlmodel
esy esy
redis redis
psycopg2-binary