From 84b7d44c4278f27dc52c2002fa30fbb7289a434e Mon Sep 17 00:00:00 2001 From: Tom Villette Date: Thu, 17 Aug 2023 18:17:34 +0200 Subject: [PATCH] switching to sqlmodel... --- Dockerfile | 3 +- eveal/__init__.py | 0 eveal/database.py | 14 +++++ main.py => eveal/main.py | 29 ++++----- eveal/models_sde.py | 109 +++++++++++++++++++++++++++++++++ schemas.py => eveal/schemas.py | 0 import_sde.py | 109 +++++++++++++++++++++++++++++++++ requirements.txt | 3 +- 8 files changed, 250 insertions(+), 17 deletions(-) create mode 100644 eveal/__init__.py create mode 100644 eveal/database.py rename main.py => eveal/main.py (60%) create mode 100644 eveal/models_sde.py rename schemas.py => eveal/schemas.py (100%) create mode 100644 import_sde.py diff --git a/Dockerfile b/Dockerfile index cbdb117..ca04d8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,7 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser WORKDIR /app COPY requirements.txt . RUN python -m pip install --no-cache-dir --upgrade -r requirements.txt -COPY --chown=appuser:appuser *.py . -COPY --chown=appuser:appuser invTypeMaterials.csv . +COPY --chown=appuser:appuser eveal /app USER appuser CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/eveal/__init__.py b/eveal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eveal/database.py b/eveal/database.py new file mode 100644 index 0000000..a9e9462 --- /dev/null +++ b/eveal/database.py @@ -0,0 +1,14 @@ +from sqlmodel import create_engine, SQLModel, Session + +from eveal import models_sde + +sqlite_file_name = "eveal.db" +sqlite_url = f"sqlite:///{sqlite_file_name}" +engine = create_engine(sqlite_url, echo=True, future=True) + +def get_session(): + db = Session(engine) + try: + yield db + finally: + db.close() diff --git a/main.py b/eveal/main.py similarity index 60% rename from main.py rename to eveal/main.py index e78f860..e35dac1 100644 --- a/main.py +++ b/eveal/main.py @@ -1,20 +1,19 @@ from collections import defaultdict -from fastapi import FastAPI +from fastapi import FastAPI, Depends from fastapi.middleware.cors import CORSMiddleware from typing import List -from schemas import Evepraisal, PriceReprocess -import csv +from sqlmodel import SQLModel, Session, select + +from eveal.schemas import Evepraisal, PriceReprocess +from eveal.database import engine, get_session +from eveal import models_sde + +SQLModel.metadata.create_all(engine) app = FastAPI() app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"]) -invtypematerials = defaultdict(dict) -with open("invTypeMaterials.csv") as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - invtypematerials[int(row['typeID'])].update({int(row['materialTypeID']): int(row['quantity'])}) - @app.get("/") async def root(): @@ -22,15 +21,17 @@ async def root(): @app.post("/reprocess/") -async def reprocess(ep_items: Evepraisal, ep_mat: Evepraisal, efficiency: float = .55): +async def reprocess(ep_items: Evepraisal, ep_mat: Evepraisal, efficiency: float = .55, db: Session = Depends(get_session)): matprices = {item.typeID: {'sell': item.prices.sell.min, 'buy': item.prices.buy.max} for item in ep_mat.items} item_reprocess: List[PriceReprocess] = [] - for item in ep_items.items: + for rawitem in ep_items.items: + # item = db.exec(select(models_sde.Type).where(models_sde.Type.id == rawitem.typeID)).one() + item = db.get(models_sde.Type, rawitem.typeID) buy_reprocess = sell_reprocess = 0.0 - for mat in invtypematerials[item.typeID]: - buy_reprocess += matprices[mat]['buy'] * invtypematerials[item.typeID][mat] * efficiency - sell_reprocess += matprices[mat]['sell'] * invtypematerials[item.typeID][mat] * efficiency + for mat in item.materials.all(): + buy_reprocess += matprices[mat.type.id]['buy'] * mat.quantity * efficiency + sell_reprocess += matprices[mat.type.id]['sell'] * mat.quantity * efficiency item_reprocess.append(PriceReprocess(typeID=item.typeID, buy=item.prices.buy.max, sell=item.prices.sell.min, diff --git a/eveal/models_sde.py b/eveal/models_sde.py new file mode 100644 index 0000000..06e59e1 --- /dev/null +++ b/eveal/models_sde.py @@ -0,0 +1,109 @@ +from typing import Optional, List +from sqlmodel import SQLModel, Field, Relationship + + +class Icon(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + + description: Optional[str] = None + iconFile: str + + categories: List['Category'] = Relationship(back_populates="icon") + groups: List['Group'] = Relationship(back_populates="icon") + marketgroups: List['MarketGroup'] = Relationship(back_populates="icon") + types: List['Type'] = Relationship(back_populates="icon") + + +class Category(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + + icon_id: Optional[int] = Field(default=None, foreign_key="icon.id") + icon: Optional[Icon] = Relationship(back_populates="categories") + + name: str + published: bool + + groups: List['Group'] = Relationship(back_populates="category") + + +class Group(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + + anchorable: bool + anchored: bool + + category_id: Optional[int] = Field(default=None, foreign_key="category.id") + category: Optional[Category] = Relationship(back_populates="groups") + + fittableNonSingletion: bool + + icon_id: Optional[int] = Field(default=None, foreign_key="icon.id") + icon: Optional[Icon] = Relationship(back_populates="groups") + + name: str + published: bool + useBasePrice: bool + + types: List['Type'] = Relationship(back_populates="group") + + +class MarketGroup(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + + description: Optional[str] = None + hasTypes: bool + + icon_id: Optional[int] = Field(default=None, foreign_key="icon.id") + icon: Optional[Icon] = Relationship(back_populates="marketgroups") + + name: str + + parent_marketgroup_id: Optional[int] = Field(default=None, foreign_key="marketgroup.id") + parent_marketgroup: Optional['MarketGroup'] = Relationship(back_populates="children_marketgroups", + sa_relationship_kwargs={"remote_side": 'MarketGroup.id'}) # workaround for self reference: https://github.com/tiangolo/sqlmodel/issues/127#issuecomment-1224135123 + children_marketgroups: List['MarketGroup'] = Relationship(back_populates="parent_marketgroup") + + types: List['Type'] = Relationship(back_populates="marketgroup") + + +class Type(SQLModel, table=True): + id: int = Field(primary_key=True) + + # id = Column(Integer, primary_key=True, index=True) + group_id: Optional[int] = Field(default=None, foreign_key="group.id") + group: Optional[Group] = Relationship(back_populates="types") + + marketgroup_id: Optional[int] = Field(default=None, foreign_key="marketgroup.id") + marketgroup: Optional[MarketGroup] = Relationship(back_populates="types") + + name: str + published: bool = False + description: Optional[str] = None + basePrice: float + + icon_id: Optional[int] = Field(default=None, foreign_key="icon.id") + icon: Optional[Icon] = Relationship(back_populates="types") + + volume: Optional[float] = None + portionSize: int + + materials: List['TypeMaterial'] = Relationship(back_populates="type", + sa_relationship_kwargs={"foreign_keys": '[TypeMaterial.type_id]'}) # https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1537445078 + material_of: List['TypeMaterial'] = Relationship(back_populates="material_type", + sa_relationship_kwargs={"foreign_keys": '[TypeMaterial.material_type_id]'}) # https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1537445078 + + +class TypeMaterial(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + + type_id: Optional[int] = Field(default=None, foreign_key="type.id") + type: Optional[Type] = Relationship(back_populates="materials", + sa_relationship_kwargs={"primaryjoin": 'TypeMaterial.type_id==Type.id', + 'lazy': 'joined'}) # workaround: https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1002835506 + + material_type_id: Optional[int] = Field(default=None, foreign_key="type.id") + material_type: Optional[Type] = Relationship(back_populates="material_of", + sa_relationship_kwargs={"primaryjoin": 'TypeMaterial.material_type_id==Type.id', + 'lazy': 'joined'}) # workaround: https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1002835506 + + quantity: int diff --git a/schemas.py b/eveal/schemas.py similarity index 100% rename from schemas.py rename to eveal/schemas.py diff --git a/import_sde.py b/import_sde.py new file mode 100644 index 0000000..678c123 --- /dev/null +++ b/import_sde.py @@ -0,0 +1,109 @@ +import yaml +from eveal.database import engine +from sqlmodel import Session, SQLModel +from eveal import models_sde + +import os +os.remove("eveal.db") +SQLModel.metadata.create_all(engine) + + +print("Importing SDE data...") +print("Importing icons...") +with open("static_eve/sde/fsd/iconIDs.yaml", "r") as f: + icons = yaml.safe_load(f) + +with Session(engine) as db: + for id, icon in icons.items(): + db.add(models_sde.Icon(id=id, **icon)) + db.commit() + +print("Importing categories...") +with open("static_eve/sde/fsd/categoryIDs.yaml", "r") as f: + categories = yaml.safe_load(f) + +with Session(engine) as db: + for id, category in categories.items(): + db.add(models_sde.Category(id=id, + icon_id=category['iconID'] if 'iconID' in category else None, + name=category['name']['en'], + published=category['published'])) + db.commit() + + +print("Importing groups...") +with open("static_eve/sde/fsd/groupIDs.yaml", "r") as f: + groups = yaml.safe_load(f) + +with Session(engine) as db: + for id, group in groups.items(): + db.add(models_sde.Group(id=id, + anchorable=group['anchorable'], + anchored=group['anchored'], + category_id=group['categoryID'], + fittableNonSingletion=group['fittableNonSingleton'], + icon_id=group['iconID'] if 'iconID' in group else None, + name=group['name']['en'], + published=group['published'], + useBasePrice=group['useBasePrice'] + )) + db.commit() + + +print("Importing marketgroups...") +with open("static_eve/sde/fsd/marketGroups.yaml", "r") as f: + marketgroups = yaml.safe_load(f) + + +with Session(engine) as db: + for id, marketgroup in marketgroups.items(): + db.add(models_sde.MarketGroup(id=id, + description=marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None, + hasTypes=marketgroup['hasTypes'], + icon_id=marketgroup['iconID'] if 'iconID' in marketgroup else None, + name=marketgroup['nameID']['en'], + parent_marketgroup_id=marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None, + )) + db.commit() + + +print("Importing types...") +with open("static_eve/sde/fsd/typeIDs.yaml", "r") as f: + types = yaml.safe_load(f) + +with Session(engine) as db: + for id, type in types.items(): + try: + db.add(models_sde.Type(id=id, + group_id=type['groupID'], + marketgroup_id=type['marketGroupID'] if 'marketGroupID' in type else None, + name=type['name']['en'], + published=type['published'], + basePrice=type['basePrice'] if 'basePrice' in type else None, + description=type['description']['en'] if 'description' in type else None, + icon_id=type['iconID'] if 'iconID' in type else None, + portionSize=type['portionSize'], + volume=type['volume'] if 'volume' in type else None, + )) + except KeyError: + print(type) + raise + db.commit() + + +print("Importing materials...") +with open("static_eve/sde/fsd/typeMaterials.yaml", "r") as f: + materials = yaml.safe_load(f) + +with Session(engine) as db: + for id, material in materials.items(): + for mat in material: + db.add(models_sde.TypeMaterial(type_id=id, + material_id=mat['materialTypeID'], + quantity=mat['quantity'] + )) + db.commit() + + + +print("DONE!") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d3f6d75..5760613 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ fastapi httpx -uvicorn[standard] \ No newline at end of file +uvicorn[standard] +sqlmodel \ No newline at end of file