switching to sqlmodel...

Dockerfile

@@ -8,8 +8,7 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser
 WORKDIR /app
 COPY requirements.txt .
 RUN python -m pip install --no-cache-dir --upgrade -r requirements.txt
-COPY --chown=appuser:appuser *.py .
+COPY --chown=appuser:appuser eveal /app
-COPY --chown=appuser:appuser invTypeMaterials.csv .
 
 USER appuser
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

eveal/database.py

@@ -0,0 +1,14 @@
+from sqlmodel import create_engine, SQLModel, Session
+
+from eveal import models_sde
+
+sqlite_file_name = "eveal.db"
+sqlite_url = f"sqlite:///{sqlite_file_name}"
+engine = create_engine(sqlite_url, echo=True, future=True)
+
+def get_session():
+    db = Session(engine)
+    try:
+        yield db
+    finally:
+        db.close()

eveal/main.py

@@ -1,20 +1,19 @@
 from collections import defaultdict
 
-from fastapi import FastAPI
+from fastapi import FastAPI, Depends
 from fastapi.middleware.cors import CORSMiddleware
 from typing import List
-from schemas import Evepraisal, PriceReprocess
+from sqlmodel import SQLModel, Session, select
-import csv
+
+from eveal.schemas import Evepraisal, PriceReprocess
+from eveal.database import engine, get_session
+from eveal import models_sde
+
+SQLModel.metadata.create_all(engine)
 
 app = FastAPI()
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
 
-invtypematerials = defaultdict(dict)
-with open("invTypeMaterials.csv") as csvfile:
-    reader = csv.DictReader(csvfile)
-    for row in reader:
-        invtypematerials[int(row['typeID'])].update({int(row['materialTypeID']): int(row['quantity'])})
-
 
 @app.get("/")
 async def root():
@@ -22,15 +21,17 @@ async def root():
 
 
 @app.post("/reprocess/")
-async def reprocess(ep_items: Evepraisal, ep_mat: Evepraisal, efficiency: float = .55):
+async def reprocess(ep_items: Evepraisal, ep_mat: Evepraisal, efficiency: float = .55, db: Session = Depends(get_session)):
     matprices = {item.typeID: {'sell': item.prices.sell.min, 'buy': item.prices.buy.max} for item in ep_mat.items}
 
     item_reprocess: List[PriceReprocess] = []
-    for item in ep_items.items:
+    for rawitem in ep_items.items:
+        # item = db.exec(select(models_sde.Type).where(models_sde.Type.id == rawitem.typeID)).one()
+        item = db.get(models_sde.Type, rawitem.typeID)
         buy_reprocess = sell_reprocess = 0.0
-        for mat in invtypematerials[item.typeID]:
+        for mat in item.materials.all():
-            buy_reprocess += matprices[mat]['buy'] * invtypematerials[item.typeID][mat] * efficiency
+            buy_reprocess += matprices[mat.type.id]['buy'] * mat.quantity * efficiency
-            sell_reprocess += matprices[mat]['sell'] * invtypematerials[item.typeID][mat] * efficiency
+            sell_reprocess += matprices[mat.type.id]['sell'] * mat.quantity * efficiency
         item_reprocess.append(PriceReprocess(typeID=item.typeID,
                                              buy=item.prices.buy.max,
                                              sell=item.prices.sell.min,

eveal/models_sde.py

@@ -0,0 +1,109 @@
+from typing import Optional, List
+from sqlmodel import SQLModel, Field, Relationship
+
+
+class Icon(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    description: Optional[str] = None
+    iconFile: str
+
+    categories: List['Category'] = Relationship(back_populates="icon")
+    groups: List['Group'] = Relationship(back_populates="icon")
+    marketgroups: List['MarketGroup'] = Relationship(back_populates="icon")
+    types: List['Type'] = Relationship(back_populates="icon")
+
+
+class Category(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    icon_id: Optional[int] = Field(default=None, foreign_key="icon.id")
+    icon: Optional[Icon] = Relationship(back_populates="categories")
+
+    name: str
+    published: bool
+
+    groups: List['Group'] = Relationship(back_populates="category")
+
+
+class Group(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    anchorable: bool
+    anchored: bool
+
+    category_id: Optional[int] = Field(default=None, foreign_key="category.id")
+    category: Optional[Category] = Relationship(back_populates="groups")
+
+    fittableNonSingletion: bool
+
+    icon_id: Optional[int] = Field(default=None, foreign_key="icon.id")
+    icon: Optional[Icon] = Relationship(back_populates="groups")
+
+    name: str
+    published: bool
+    useBasePrice: bool
+
+    types: List['Type'] = Relationship(back_populates="group")
+
+
+class MarketGroup(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    description: Optional[str] = None
+    hasTypes: bool
+
+    icon_id: Optional[int] = Field(default=None, foreign_key="icon.id")
+    icon: Optional[Icon] = Relationship(back_populates="marketgroups")
+
+    name: str
+
+    parent_marketgroup_id: Optional[int] = Field(default=None, foreign_key="marketgroup.id")
+    parent_marketgroup: Optional['MarketGroup'] = Relationship(back_populates="children_marketgroups",
+                                                               sa_relationship_kwargs={"remote_side": 'MarketGroup.id'})  # workaround for self reference: https://github.com/tiangolo/sqlmodel/issues/127#issuecomment-1224135123
+    children_marketgroups: List['MarketGroup'] = Relationship(back_populates="parent_marketgroup")
+
+    types: List['Type'] = Relationship(back_populates="marketgroup")
+
+
+class Type(SQLModel, table=True):
+    id: int = Field(primary_key=True)
+
+    # id = Column(Integer, primary_key=True, index=True)
+    group_id: Optional[int] = Field(default=None, foreign_key="group.id")
+    group: Optional[Group] = Relationship(back_populates="types")
+
+    marketgroup_id: Optional[int] = Field(default=None, foreign_key="marketgroup.id")
+    marketgroup: Optional[MarketGroup] = Relationship(back_populates="types")
+
+    name: str
+    published: bool = False
+    description: Optional[str] = None
+    basePrice: float
+
+    icon_id: Optional[int] = Field(default=None, foreign_key="icon.id")
+    icon: Optional[Icon] = Relationship(back_populates="types")
+
+    volume: Optional[float] = None
+    portionSize: int
+
+    materials: List['TypeMaterial'] = Relationship(back_populates="type",
+                                                   sa_relationship_kwargs={"foreign_keys": '[TypeMaterial.type_id]'})  # https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1537445078
+    material_of: List['TypeMaterial'] = Relationship(back_populates="material_type",
+                                                     sa_relationship_kwargs={"foreign_keys": '[TypeMaterial.material_type_id]'})  # https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1537445078
+
+
+class TypeMaterial(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    type_id: Optional[int] = Field(default=None, foreign_key="type.id")
+    type: Optional[Type] = Relationship(back_populates="materials",
+                                        sa_relationship_kwargs={"primaryjoin": 'TypeMaterial.type_id==Type.id',
+                                                                'lazy': 'joined'})  # workaround: https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1002835506
+
+    material_type_id: Optional[int] = Field(default=None, foreign_key="type.id")
+    material_type: Optional[Type] = Relationship(back_populates="material_of",
+                                                 sa_relationship_kwargs={"primaryjoin": 'TypeMaterial.material_type_id==Type.id',
+                                                                         'lazy': 'joined'})  # workaround: https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1002835506
+
+    quantity: int

import_sde.py

@@ -0,0 +1,109 @@
+import yaml
+from eveal.database import engine
+from sqlmodel import Session, SQLModel
+from eveal import models_sde
+
+import os
+os.remove("eveal.db")
+SQLModel.metadata.create_all(engine)
+
+
+print("Importing SDE data...")
+print("Importing icons...")
+with open("static_eve/sde/fsd/iconIDs.yaml", "r") as f:
+    icons = yaml.safe_load(f)
+
+with Session(engine) as db:
+    for id, icon in icons.items():
+        db.add(models_sde.Icon(id=id, **icon))
+    db.commit()
+
+print("Importing categories...")
+with open("static_eve/sde/fsd/categoryIDs.yaml", "r") as f:
+    categories = yaml.safe_load(f)
+
+with Session(engine) as db:
+    for id, category in categories.items():
+        db.add(models_sde.Category(id=id,
+                                   icon_id=category['iconID'] if 'iconID' in category else None,
+                                   name=category['name']['en'],
+                                   published=category['published']))
+    db.commit()
+
+
+print("Importing groups...")
+with open("static_eve/sde/fsd/groupIDs.yaml", "r") as f:
+    groups = yaml.safe_load(f)
+
+with Session(engine) as db:
+    for id, group in groups.items():
+        db.add(models_sde.Group(id=id,
+                                anchorable=group['anchorable'],
+                                anchored=group['anchored'],
+                                category_id=group['categoryID'],
+                                fittableNonSingletion=group['fittableNonSingleton'],
+                                icon_id=group['iconID'] if 'iconID' in group else None,
+                                name=group['name']['en'],
+                                published=group['published'],
+                                useBasePrice=group['useBasePrice']
+                                ))
+    db.commit()
+
+
+print("Importing marketgroups...")
+with open("static_eve/sde/fsd/marketGroups.yaml", "r") as f:
+    marketgroups = yaml.safe_load(f)
+
+
+with Session(engine) as db:
+    for id, marketgroup in marketgroups.items():
+        db.add(models_sde.MarketGroup(id=id,
+                                      description=marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None,
+                                      hasTypes=marketgroup['hasTypes'],
+                                      icon_id=marketgroup['iconID'] if 'iconID' in marketgroup else None,
+                                      name=marketgroup['nameID']['en'],
+                                      parent_marketgroup_id=marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None,
+                                      ))
+    db.commit()
+
+
+print("Importing types...")
+with open("static_eve/sde/fsd/typeIDs.yaml", "r") as f:
+    types = yaml.safe_load(f)
+
+with Session(engine) as db:
+    for id, type in types.items():
+        try:
+            db.add(models_sde.Type(id=id,
+                                   group_id=type['groupID'],
+                                   marketgroup_id=type['marketGroupID'] if 'marketGroupID' in type else None,
+                                   name=type['name']['en'],
+                                   published=type['published'],
+                                   basePrice=type['basePrice'] if 'basePrice' in type else None,
+                                   description=type['description']['en'] if 'description' in type else None,
+                                   icon_id=type['iconID'] if 'iconID' in type else None,
+                                   portionSize=type['portionSize'],
+                                   volume=type['volume'] if 'volume' in type else None,
+                                   ))
+        except KeyError:
+            print(type)
+            raise
+    db.commit()
+
+
+print("Importing materials...")
+with open("static_eve/sde/fsd/typeMaterials.yaml", "r") as f:
+    materials = yaml.safe_load(f)
+
+with Session(engine) as db:
+    for id, material in materials.items():
+        for mat in material:
+            db.add(models_sde.TypeMaterial(type_id=id,
+                                           material_id=mat['materialTypeID'],
+                                           quantity=mat['quantity']
+                                           ))
+    db.commit()
+
+
+
+print("DONE!")

requirements.txt

@@ -1,3 +1,4 @@
 fastapi
 httpx
-uvicorn[standard]
+uvicorn[standard]
+sqlmodel