eveal/marbas
2
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

transition to postgres

Browse Source
This commit is contained in:
Tom Villette
2023-10-12 18:42:17 +02:00
parent 4ce0b80196
commit e639f12f96
6 changed files with 128 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.env.template
View File

@@ -5,12 +5,10 @@ ESI_USER_AGENT=
REDIS_URL=
REDIS_PORT=
REDIS_USER=
REDIS_PASSWORD=
REDIS_SSL=
SQLITE_DB_PATH=
POSTGRES_HOST=
POSTGRES_PASSWORD=
POSTGRES_USER=
POSTGRES_DB=

41
docker-compose.dev.yml
View File

@@ -1,7 +1,7 @@
version: '3'
services:
eveal:
image: eveal:latest
image: mabras:local
build:
context: .
dockerfile: Dockerfile
@@ -11,13 +11,12 @@ services:
- 8000:8000
volumes:
- ./eveal:/app/eveal
- ./eveal.db:/app/eveal.db
command: ["uvicorn", "eveal.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
depends_on:
redis:
condition: service_healthy
# db:
# condition: service_healthy
db:
condition: service_healthy
# elasticsearch:
# condition: service_healthy
@@ -29,22 +28,21 @@ services:
test: redis-cli ping
interval: 3s
# db:
# image: postgres:13-alpine
# ports:
# - 5432:5432
# volumes:
# - ./dump_sde.sql:/docker-entrypoint-initdb.d/init_sde.sql
# - mabras_dbdata:/var/lib/postgresql/data
# environment:
# - POSTGRES_PASSWORD
# - POSTGRES_USER
# - POSTGRES_DB
# healthcheck:
# test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}
# interval: 10s
# timeout: 3s
# retries: 3
db:
image: postgres:13-alpine
ports:
- 5432:5432
volumes:
- mabras_dbdata:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD
- POSTGRES_USER
- POSTGRES_DB
healthcheck:
test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}
interval: 10s
timeout: 3s
retries: 3
# elasticsearch:
# image: elasticsearch:latest
@@ -60,3 +58,6 @@ services:
# interval: 10s
# timeout: 3s
# retries: 3
volumes:
mabras_dbdata:

9
eveal/database.py
View File

@@ -1,10 +1,13 @@
import os
from sqlmodel import create_engine, Session
from eveal import models_sde
sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db")
engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=True, future=True, connect_args={"check_same_thread": False})
if os.getenv("POSTGRES_HOST"):
engine = create_engine(f"postgresql://{os.getenv('POSTGRES_USER')}:{os.getenv('POSTGRES_PASSWORD')}@{os.getenv('POSTGRES_HOST')}/{os.getenv('POSTGRES_DB')}",
echo=False, future=True)
else:
sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db")
engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=False, future=True, connect_args={"check_same_thread": False})
def get_session():
db = Session(engine)

7
eveal/esi.py
View File

@@ -7,8 +7,8 @@ from esy.auth import ESIAuthenticator
class ESICache(object):
def __init__(self, redis_url: str, redis_port: int, db: str):
self._r = redis.Redis(host=redis_url, port=redis_port, db=db)
def __init__(self, **kwargs):
self._r = redis.Redis(**kwargs)
# self._r = redis.StrictRedis(host=redis_url, port=redis_port, db=db)
def get(self, key):
@@ -26,7 +26,8 @@ class ESICache(object):
esi_client_id = os.getenv('ESI_CLIENT_ID')
esi_secret_key = os.getenv('ESI_SECRET_KEY')
esi_cache = ESICache(redis_url=os.getenv("REDIS_URL"), redis_port=int(os.getenv("REDIS_PORT")), db="0")
esi_cache = ESICache(host=os.getenv("REDIS_URL"), port=int(os.getenv("REDIS_PORT")), db="0",
password=os.getenv("REDIS_PASSWD"))
esi_client = ESIClient.get_client(user_agent=os.getenv('ESI_USER_AGENT'), cache=esi_cache)
esi_auth = ESIAuthenticator()

151
import_sde.py
View File

@@ -1,13 +1,8 @@
import yaml
from eveal.database import engine, sqlite_file_name
from sqlmodel import Session, SQLModel
from eveal.database import engine
from sqlmodel import Session, SQLModel, select
from eveal import models_sde
import os
try:
os.remove(sqlite_file_name)
except:
pass
SQLModel.metadata.create_all(engine)
@@ -15,99 +10,143 @@ print("Importing SDE data...")
print("Importing icons...")
with open("static_eve/sde/fsd/iconIDs.yaml", "r", encoding="utf-8") as f:
icons = yaml.safe_load(f)
new_icons = total_icons = 0
with Session(engine) as db:
for id, icon in icons.items():
db.add(models_sde.SDEIcon(id=id, **icon))
db_icon = db.get(models_sde.SDEIcon, id)
if not db_icon:
db_icon = models_sde.SDEIcon(id=id)
db.add(db_icon)
new_icons += 1
db_icon.iconFile = icon['iconFile']
db_icon.description = icon['description'] if 'description' in icon else None
total_icons += 1
db.commit()
print(f"Imported {new_icons} new icons / {total_icons} total icons.")
print("Importing categories...")
with open("static_eve/sde/fsd/categoryIDs.yaml", "r", encoding="utf-8") as f:
categories = yaml.safe_load(f)
new_categories = total_categories = 0
with Session(engine) as db:
for id, category in categories.items():
if category["published"] == False:
continue
db.add(models_sde.SDECategory(id=id,
icon_id=category['iconID'] if 'iconID' in category else None,
name=category['name']['en'],
published=category['published']))
# if category["published"] == False:
# continue
db_category = db.get(models_sde.SDECategory, id)
if not db_category:
db_category = models_sde.SDECategory(id=id)
db.add(db_category)
new_categories += 1
db_category.name = category['name']['en']
db_category.published = category['published']
db_category.icon_id = category['iconID'] if 'iconID' in category else None
total_categories += 1
db.commit()
print(f"Imported {new_categories} new categories / {total_categories} total categories.")
print("Importing groups...")
with open("static_eve/sde/fsd/groupIDs.yaml", "r", encoding="utf-8") as f:
groups = yaml.safe_load(f)
new_groups = total_groups = 0
with Session(engine) as db:
for id, group in groups.items():
if group["published"] == False:
continue
db.add(models_sde.SDEGroup(id=id,
anchorable=group['anchorable'],
anchored=group['anchored'],
category_id=group['categoryID'],
fittableNonSingletion=group['fittableNonSingleton'],
icon_id=group['iconID'] if 'iconID' in group else None,
name=group['name']['en'],
published=group['published'],
useBasePrice=group['useBasePrice']
))
# if group["published"] == False:
# continue
db_group = db.get(models_sde.SDEGroup, id)
if not db_group:
db_group = models_sde.SDEGroup(id=id)
db.add(db_group)
new_groups += 1
db_group.anchorable = group['anchorable']
db_group.anchored = group['anchored']
db_group.category_id = group['categoryID']
db_group.fittableNonSingletion = group['fittableNonSingleton']
db_group.icon_id = group['iconID'] if 'iconID' in group else None
db_group.name = group['name']['en']
db_group.published = group['published']
db_group.useBasePrice = group['useBasePrice']
total_groups += 1
db.commit()
print(f"Imported {new_groups} new groups / {total_groups} total groups.")
print("Importing marketgroups...")
with open("static_eve/sde/fsd/marketGroups.yaml", "r", encoding="utf-8") as f:
marketgroups = yaml.safe_load(f)
new_marketgroups = total_marketgroups = 0
with Session(engine) as db:
for id, marketgroup in marketgroups.items():
db.add(models_sde.SDEMarketGroup(id=id,
description=marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None,
hasTypes=marketgroup['hasTypes'],
icon_id=marketgroup['iconID'] if 'iconID' in marketgroup else None,
name=marketgroup['nameID']['en'],
parent_marketgroup_id=marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None,
))
db_marketgroups = db.get(models_sde.SDEMarketGroup, id)
if not db_marketgroups:
db_marketgroups = models_sde.SDEMarketGroup(id=id)
db.add(db_marketgroups)
new_marketgroups += 1
db_marketgroups.description = marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None
db_marketgroups.hasTypes = marketgroup['hasTypes']
db_marketgroups.icon_id = marketgroup['iconID'] if 'iconID' in marketgroup else None
db_marketgroups.name = marketgroup['nameID']['en']
# db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None
total_marketgroups += 1
db.commit()
print(f"Imported {new_marketgroups} marketgroups / {total_marketgroups} total marketgroups.")
print("Setting up marketgroup Parents...")
total_marketgroup_links = 0
with Session(engine) as db:
for id, marketgroup in marketgroups.items():
db_marketgroups = db.get(models_sde.SDEMarketGroup, id)
db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None
total_marketgroup_links += 1
db.commit()
print(f"Updated {total_marketgroup_links} marketgroup Parents")
print("Importing types...")
with open("static_eve/sde/fsd/typeIDs.yaml", "r", encoding="utf-8") as f:
types = yaml.safe_load(f)
new_types = total_types = 0
with Session(engine) as db:
for id, type in types.items():
if type["published"] == False:
continue
db.add(models_sde.SDEType(id=id,
group_id=type['groupID'],
marketgroup_id=type['marketGroupID'] if 'marketGroupID' in type else None,
name=type['name']['en'],
published=type['published'],
basePrice=type['basePrice'] if 'basePrice' in type else None,
description=type['description']['en'] if 'description' in type else None,
icon_id=type['iconID'] if 'iconID' in type else None,
portionSize=type['portionSize'],
volume=type['volume'] if 'volume' in type else None,
))
# if type["published"] == False:
# continue
db_type = db.get(models_sde.SDEType, id)
if not db_type:
db_type = models_sde.SDEType(id=id)
db.add(db_type)
new_types += 1
db_type.group_id = type['groupID']
db_type.marketgroup_id = type['marketGroupID'] if 'marketGroupID' in type else None
db_type.name = type['name']['en']
db_type.published = type['published']
db_type.basePrice = type['basePrice'] if 'basePrice' in type else None
db_type.description = type['description']['en'] if 'description' in type else None
db_type.icon_id = type['iconID'] if 'iconID' in type else None
db_type.portionSize = type['portionSize']
db_type.volume = type['volume'] if 'volume' in type else None
total_types += 1
db.commit()
print(f"Imported {new_types} types / {total_types} total types.")
print("Importing materials...")
with open("static_eve/sde/fsd/typeMaterials.yaml", "r", encoding="utf-8") as f:
materials = yaml.safe_load(f)
new_typemat = total_typemat = 0
with Session(engine) as db:
for id, material in materials.items():
for mat in material['materials']:
db.add(models_sde.SDETypeMaterial(type_id=id,
material_id=mat['materialTypeID'],
quantity=mat['quantity']
))
db_typemat = db.exec(select(models_sde.SDETypeMaterial).where(models_sde.SDETypeMaterial.type_id == id, models_sde.SDETypeMaterial.material_type_id == mat['materialTypeID'])).one_or_none()
if not db_typemat:
db_typemat = models_sde.SDETypeMaterial()
db.add(db_typemat)
new_typemat += 1
db_typemat.type_id = id
db_typemat.material_type_id = mat['materialTypeID']
db_typemat.quantity = mat['quantity']
total_typemat += 1
db.commit()
print(f"Imported {new_typemat} materials / {total_typemat} total materials.")
print("DONE!")

1
requirements.txt
View File

@@ -4,3 +4,4 @@ uvicorn[standard]
sqlmodel
esy
redis
psycopg2-binary