From d4c09332b5472302f8f7f38e4eec5a5514f78daf Mon Sep 17 00:00:00 2001 From: Tom Villette Date: Thu, 26 Oct 2023 18:05:30 +0200 Subject: [PATCH 01/13] init DRF + sde models --- .env.template | 3 + Dockerfile | 6 +- api/__init__.py | 0 api/admin.py | 3 + api/apps.py | 6 ++ {eveal => api}/esi.py | 0 api/migrations/0001_initial.py | 108 +++++++++++++++++++++++++ api/migrations/__init__.py | 0 api/models.py | 4 + api/models_esi.py | 2 + api/models_sde.py | 66 +++++++++++++++ api/serializers.py | 15 ++++ api/tests.py | 3 + api/views.py | 24 ++++++ docker-compose.dev.yml | 29 +++++-- eveal/main.py | 14 ++-- import_sde.py | 7 -- mabras/__init__.py | 0 mabras/asgi.py | 16 ++++ mabras/settings.py | 141 +++++++++++++++++++++++++++++++++ mabras/urls.py | 31 ++++++++ mabras/wsgi.py | 16 ++++ manage.py | 22 +++++ requirements.txt | 11 ++- 24 files changed, 500 insertions(+), 27 deletions(-) create mode 100644 api/__init__.py create mode 100644 api/admin.py create mode 100644 api/apps.py rename {eveal => api}/esi.py (100%) create mode 100644 api/migrations/0001_initial.py create mode 100644 api/migrations/__init__.py create mode 100644 api/models.py create mode 100644 api/models_esi.py create mode 100644 api/models_sde.py create mode 100644 api/serializers.py create mode 100644 api/tests.py create mode 100644 api/views.py create mode 100644 mabras/__init__.py create mode 100644 mabras/asgi.py create mode 100644 mabras/settings.py create mode 100644 mabras/urls.py create mode 100644 mabras/wsgi.py create mode 100755 manage.py diff --git a/.env.template b/.env.template index c5f00e2..fb45ed3 100644 --- a/.env.template +++ b/.env.template @@ -12,3 +12,6 @@ POSTGRES_HOST= POSTGRES_PASSWORD= POSTGRES_USER= POSTGRES_DB= + +DRF_SECRET_KEY= +DRF_DEBUG= \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ee2f1c1..e126bfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,11 @@ ENV PYTHONDONTWRITEBYTECODE=1 RUN adduser -u 5678 --disabled-password --gecos "" appuser WORKDIR /app +COPY --chown=appuser:appuser manage.py /app/manage.py COPY requirements.txt . RUN python -m pip install --no-cache-dir --upgrade -r requirements.txt -COPY --chown=appuser:appuser eveal /app/eveal +COPY --chown=appuser:appuser mabras /app/mabras +COPY --chown=appuser:appuser api /app/api USER appuser -CMD ["uvicorn", "eveal.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/admin.py b/api/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/api/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/api/apps.py b/api/apps.py new file mode 100644 index 0000000..66656fd --- /dev/null +++ b/api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'api' diff --git a/eveal/esi.py b/api/esi.py similarity index 100% rename from eveal/esi.py rename to api/esi.py diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py new file mode 100644 index 0000000..77f2d3a --- /dev/null +++ b/api/migrations/0001_initial.py @@ -0,0 +1,108 @@ +# Generated by Django 4.2.6 on 2023-10-26 15:51 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SDECategory', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('published', models.BooleanField()), + ], + ), + migrations.CreateModel( + name='SDEGroup', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('published', models.BooleanField()), + ('useBasePrice', models.BooleanField()), + ('fittableNonSingletion', models.BooleanField()), + ('anchored', models.BooleanField()), + ('anchorable', models.BooleanField()), + ('category', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='api.sdecategory')), + ], + ), + migrations.CreateModel( + name='SDEIcon', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('iconFile', models.CharField()), + ('description', models.CharField()), + ], + ), + migrations.CreateModel( + name='SDEMarektGroup', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('description', models.CharField(default='')), + ('hasTypes', models.BooleanField()), + ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.sdeicon')), + ('parent_marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='child_marketgroups', to='api.sdemarektgroup')), + ], + ), + migrations.CreateModel( + name='SDEMetaGroup', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('iconSuffix', models.CharField()), + ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='metagroups', to='api.sdeicon')), + ], + ), + migrations.CreateModel( + name='SDEType', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('description', models.CharField()), + ('published', models.BooleanField()), + ('basePrice', models.FloatField()), + ('volume', models.FloatField()), + ('portionSize', models.IntegerField()), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdegroup')), + ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdeicon')), + ('marketgroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdemarektgroup')), + ], + ), + migrations.CreateModel( + name='SDETypeMaterial', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('quantity', models.IntegerField()), + ('material_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials_of', to='api.sdetype')), + ('type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials', to='api.sdetype')), + ], + ), + migrations.AddField( + model_name='sdetype', + name='materials', + field=models.ManyToManyField(related_name='material_of', through='api.SDETypeMaterial', to='api.sdetype'), + ), + migrations.AddField( + model_name='sdetype', + name='metagroup', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdemetagroup'), + ), + migrations.AddField( + model_name='sdegroup', + name='icon', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='groups', to='api.sdeicon'), + ), + migrations.AddField( + model_name='sdecategory', + name='icon', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='categories', to='api.sdeicon'), + ), + ] diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/models.py b/api/models.py new file mode 100644 index 0000000..fa6a91e --- /dev/null +++ b/api/models.py @@ -0,0 +1,4 @@ +from django.db import models +from .models_esi import * +from .models_sde import * + diff --git a/api/models_esi.py b/api/models_esi.py new file mode 100644 index 0000000..beeb308 --- /dev/null +++ b/api/models_esi.py @@ -0,0 +1,2 @@ +from django.db import models + diff --git a/api/models_sde.py b/api/models_sde.py new file mode 100644 index 0000000..8b3ff62 --- /dev/null +++ b/api/models_sde.py @@ -0,0 +1,66 @@ +from django.db import models + + +class SDEIcon(models.Model): + id = models.IntegerField(primary_key=True) + iconFile = models.CharField() + description = models.CharField() + + +class SDECategory(models.Model): + id = models.IntegerField(primary_key=True) + icon = models.ForeignKey(SDEIcon, related_name="categories", null=True, on_delete=models.SET_NULL) + name = models.CharField() + published = models.BooleanField() + + +class SDEGroup(models.Model): + id = models.IntegerField(primary_key=True) + category = models.ForeignKey(SDECategory, related_name="groups", on_delete=models.CASCADE) + name = models.CharField() + published = models.BooleanField() + useBasePrice = models.BooleanField() + fittableNonSingletion = models.BooleanField() + anchored = models.BooleanField() + anchorable = models.BooleanField() + icon = models.ForeignKey(SDEIcon, related_name="groups", null=True, on_delete=models.SET_NULL) + + +class SDEMarektGroup(models.Model): + id = models.IntegerField(primary_key=True) + icon = models.ForeignKey(SDEIcon, null=True, on_delete=models.SET_NULL) + name = models.CharField() + description = models.CharField(default="") + hasTypes = models.BooleanField() + parent_marketgroup = models.ForeignKey("self", null=True, related_name="child_marketgroups", on_delete=models.CASCADE) + + +class SDEMetaGroup(models.Model): + id = models.IntegerField(primary_key=True) + icon = models.ForeignKey(SDEIcon, related_name="metagroups", null=True, on_delete=models.SET_NULL) + name = models.CharField() + iconSuffix = models.CharField() + + +class SDEType(models.Model): + id = models.IntegerField(primary_key=True) + group = models.ForeignKey(SDEGroup, related_name="types", on_delete=models.CASCADE) + marketgroup = models.ForeignKey(SDEMarektGroup, related_name="types", on_delete=models.CASCADE) + metagroup = models.ForeignKey(SDEMetaGroup, related_name="types", on_delete=models.CASCADE) + name = models.CharField() + description = models.CharField() + published = models.BooleanField() + basePrice = models.FloatField() + icon = models.ForeignKey(SDEIcon, related_name="types", null=True, on_delete=models.SET_NULL) + volume = models.FloatField() + portionSize = models.IntegerField() + materials = models.ManyToManyField("self", through="SDETypeMaterial", symmetrical=False, related_name="material_of") + + +class SDETypeMaterial(models.Model): + id = models.IntegerField(primary_key=True) + type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials") + material_type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials_of") + + quantity = models.IntegerField() + diff --git a/api/serializers.py b/api/serializers.py new file mode 100644 index 0000000..466c3f6 --- /dev/null +++ b/api/serializers.py @@ -0,0 +1,15 @@ +from django.contrib.auth.models import User, Group +from rest_framework import serializers + + +class UserSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = User + fields = ['url', 'username', 'email', 'groups'] + + +class GroupSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Group + fields = ['url', 'name'] + diff --git a/api/tests.py b/api/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/api/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/api/views.py b/api/views.py new file mode 100644 index 0000000..4f59f72 --- /dev/null +++ b/api/views.py @@ -0,0 +1,24 @@ +from django.shortcuts import render +from django.contrib.auth.models import User, Group +from rest_framework import viewsets +from rest_framework import permissions +from api.serializers import UserSerializer, GroupSerializer + + +class UserViewSet(viewsets.ModelViewSet): + """ + API endpoint that allows users to be viewed or edited. + """ + queryset = User.objects.all().order_by('-date_joined') + serializer_class = UserSerializer + permission_classes = [permissions.IsAuthenticated] + + +class GroupViewSet(viewsets.ModelViewSet): + """ + API endpoint that allows groups to be viewed or edited. + """ + queryset = Group.objects.all() + serializer_class = GroupSerializer + permission_classes = [permissions.IsAuthenticated] + diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2e1512c..46800e9 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,22 +1,41 @@ version: '3' services: - eveal: + migrations: image: mabras:local build: context: . dockerfile: Dockerfile + env_file: + - .env + user: "1000:1000" + volumes: + - ./mabras:/app/mabras + - ./api:/app/api + - ./manage.py:/app/manage.py + command: "python manage.py makemigrations;python manage.py migrate" + depends_on: + db: + condition: service_healthy + + eveal: + image: mabras:local env_file: - .env ports: - 8000:8000 + user: "1000:1000" volumes: - - ./eveal:/app/eveal - command: ["uvicorn", "eveal.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] + - ./mabras:/app/mabras + - ./api:/app/api + - ./manage.py:/app/manage.py + command: ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000", "--reload"] depends_on: redis: condition: service_healthy db: condition: service_healthy + migrations: + condition: service_completed_successfully # elasticsearch: # condition: service_healthy @@ -38,9 +57,9 @@ services: - .env healthcheck: test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} - interval: 10s + interval: 1s timeout: 3s - retries: 3 + retries: 10 # elasticsearch: # image: elasticsearch:latest diff --git a/eveal/main.py b/eveal/main.py index 26214a7..0ec1a85 100644 --- a/eveal/main.py +++ b/eveal/main.py @@ -1,13 +1,12 @@ -from collections import defaultdict - -from fastapi import FastAPI, Depends, Path, Query +from fastapi import FastAPI, Depends from fastapi.middleware.cors import CORSMiddleware -from typing import List, Annotated, Tuple, Literal, Dict, TypedDict -from sqlmodel import SQLModel, Session, select +from typing import List, Dict +from sqlmodel import SQLModel, Session from eveal.schemas import Evepraisal, PriceReprocess from eveal.database import engine, get_session -from eveal import models_sde, esi +from eveal import models_sde +from api import esi SQLModel.metadata.create_all(engine) # use alembic? @@ -80,7 +79,8 @@ async def sde_types_search(query: List[Dict[str, int | str | None | List]], db: async def sde_types_market(sde_type: int | str, region_id: int | str, db: Session = Depends(get_session)): """Get market orders for a type in a region. example: /esi/types/22291/market/10000002/""" """TODO: use ESIMarketOrder""" - return list(esi.esi_client.Market.get_markets_region_id_orders(order_type="all", type_id=sde_type, region_id=region_id)) + return list( + esi.esi_client.Market.get_markets_region_id_orders(order_type="all", type_id=sde_type, region_id=region_id)) @app.get("/_tools/get_all_mat") diff --git a/import_sde.py b/import_sde.py index bdb8197..a827335 100644 --- a/import_sde.py +++ b/import_sde.py @@ -1,10 +1,3 @@ -import yaml -from eveal.database import engine -from sqlmodel import Session, SQLModel, select -from eveal import models_sde - -SQLModel.metadata.drop_all(engine) # use alembic! -SQLModel.metadata.create_all(engine) print("Importing SDE data...") diff --git a/mabras/__init__.py b/mabras/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mabras/asgi.py b/mabras/asgi.py new file mode 100644 index 0000000..d714762 --- /dev/null +++ b/mabras/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for mabras project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mabras.settings') + +application = get_asgi_application() diff --git a/mabras/settings.py b/mabras/settings.py new file mode 100644 index 0000000..81b72fd --- /dev/null +++ b/mabras/settings.py @@ -0,0 +1,141 @@ +""" +Django settings for mabras project. + +Generated by 'django-admin startproject' using Django 4.2.6. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +from pathlib import Path +import os + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv("DRF_SECRET_KEY") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = os.getenv("DRF_DEBUG", False) == "True" + +ALLOWED_HOSTS = [] + + +# Application definition + +REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', + 'PAGE_SIZE': 10, + 'DEFAULT_RENDERER_CLASSES': [ + 'rest_framework.renderers.JSONRenderer', + ], + 'DEFAULT_PARSER_CLASSES': [ + 'rest_framework.parsers.JSONParser', + ], +} + +INSTALLED_APPS = [ + 'api', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'mabras.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mabras.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + "HOST": os.getenv("POSTGRES_HOST"), + "PORT": os.getenv("POSTGRES_PORT", 5432), + "USER": os.getenv("POSTGRES_USER"), + "PASSWORD": os.getenv("POSTGRES_PASSWORD"), + "NAME": os.getenv("POSTGRES_DB"), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/mabras/urls.py b/mabras/urls.py new file mode 100644 index 0000000..3d54102 --- /dev/null +++ b/mabras/urls.py @@ -0,0 +1,31 @@ +""" +URL configuration for mabras project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import include, path +from rest_framework import routers +from api import views + +router = routers.DefaultRouter() +router.register(r'users', views.UserViewSet) +router.register(r'groups', views.GroupViewSet) + +# Wire up our API using automatic URL routing. +# Additionally, we include login URLs for the browsable API. +urlpatterns = [ + path('', include(router.urls)), + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')) +] \ No newline at end of file diff --git a/mabras/wsgi.py b/mabras/wsgi.py new file mode 100644 index 0000000..f15aaf8 --- /dev/null +++ b/mabras/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for mabras project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mabras.settings') + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..3a4a74c --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mabras.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 8c0c8cc..32efb56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -fastapi -httpx +django +djangorestframework +django-filter +markdown uvicorn[standard] -sqlmodel -esy -redis -psycopg2-binary +psycopg[binary] -- 2.49.1 From 06a93d92c7548f32aa6b3584ad3f5eeacdd13c98 Mon Sep 17 00:00:00 2001 From: Tom Villette Date: Thu, 26 Oct 2023 18:07:28 +0200 Subject: [PATCH 02/13] remove fastapi --- eveal/__init__.py | 0 eveal/database.py | 17 ----- eveal/main.py | 97 -------------------------- eveal/models_esi.py | 24 ------- eveal/models_sde.py | 127 --------------------------------- eveal/schemas.py | 52 -------------- import_sde.py | 166 -------------------------------------------- 7 files changed, 483 deletions(-) delete mode 100644 eveal/__init__.py delete mode 100644 eveal/database.py delete mode 100644 eveal/main.py delete mode 100644 eveal/models_esi.py delete mode 100644 eveal/models_sde.py delete mode 100644 eveal/schemas.py delete mode 100644 import_sde.py diff --git a/eveal/__init__.py b/eveal/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/eveal/database.py b/eveal/database.py deleted file mode 100644 index de94950..0000000 --- a/eveal/database.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -from sqlmodel import create_engine, Session - - -if os.getenv("POSTGRES_HOST"): - engine = create_engine(f"postgresql://{os.getenv('POSTGRES_USER')}:{os.getenv('POSTGRES_PASSWORD')}@{os.getenv('POSTGRES_HOST')}/{os.getenv('POSTGRES_DB')}", - echo=False, future=True) -else: - sqlite_file_name = os.getenv("SQLITE_DB_PATH", "eveal.db") - engine = create_engine(f"sqlite:///{sqlite_file_name}", echo=True, future=True, connect_args={"check_same_thread": False}) - -def get_session(): - db = Session(engine) - try: - yield db - finally: - db.close() diff --git a/eveal/main.py b/eveal/main.py deleted file mode 100644 index 0ec1a85..0000000 --- a/eveal/main.py +++ /dev/null @@ -1,97 +0,0 @@ -from fastapi import FastAPI, Depends -from fastapi.middleware.cors import CORSMiddleware -from typing import List, Dict -from sqlmodel import SQLModel, Session - -from eveal.schemas import Evepraisal, PriceReprocess -from eveal.database import engine, get_session -from eveal import models_sde -from api import esi - -SQLModel.metadata.create_all(engine) # use alembic? - -app = FastAPI() -app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"]) - - -@app.get("/") -async def root(): - return {"message": "Hello World"} - - -@app.post("/reprocess/") -async def reprocess(ep_items: Evepraisal, ep_mat: Evepraisal, efficiency: float = .55, db: Session = Depends(get_session)) -> List[PriceReprocess]: - matprices = {item.typeID: {'sell': item.prices.sell.min, 'buy': item.prices.buy.max} for item in ep_mat.items} - - item_reprocess: List[PriceReprocess] = [] - for rawitem in ep_items.items: - item = db.get(models_sde.SDEType, rawitem.typeID) - buy_reprocess = sell_reprocess = 0.0 - for mat in item.materials.all(): - buy_reprocess += matprices[mat.material_type_id]['buy'] * mat.quantity/mat.type.portionSize * efficiency - sell_reprocess += matprices[mat.material_type_id]['sell'] * mat.quantity/mat.type.portionSize * efficiency - item_reprocess.append(PriceReprocess(typeID=rawitem.typeID, - buy=rawitem.prices.buy.max, - sell=rawitem.prices.sell.min, - buy_reprocess=buy_reprocess, - sell_reprocess=sell_reprocess, - name=rawitem.name - )) - return item_reprocess - - -@app.get("/sde/types/{sde_type}/") -async def sde_types(sde_type: int | str, db: Session = Depends(get_session)) -> models_sde.SDEType: - try: - item = db.get(models_sde.SDEType, int(sde_type)) - except ValueError: - item = db.query(models_sde.SDEType).filter(models_sde.SDEType.name == sde_type).one() - return item - - -@app.post("/sde/types/search") -async def sde_types_search(query: List[Dict[str, int | str | None | List]], db: Session = Depends(get_session)) -> List[models_sde.SDEType]: - items = [] - for q in query: - qitems = db.query(models_sde.SDEType) - for k in q.keys(): - value = q[k] - - tokens = k.split("__") - key, mods = tokens[0], tokens[1:] - - if "i" in mods: - condition = getattr(models_sde.SDEType, key).ilike(f"%{value}%") # change to icontains when sqlmodel start using sqlalchemy > 2.0 - elif "in" in mods: - condition = getattr(models_sde.SDEType, key).in_(value) - else: - condition = getattr(models_sde.SDEType, key) == value - - if "not" in mods: - condition = ~condition - - qitems = qitems.filter(condition) - items.extend(qitems.all()) - return items - - -@app.get("/esi/types/{sde_type}/market/{region_id}/") -async def sde_types_market(sde_type: int | str, region_id: int | str, db: Session = Depends(get_session)): - """Get market orders for a type in a region. example: /esi/types/22291/market/10000002/""" - """TODO: use ESIMarketOrder""" - return list( - esi.esi_client.Market.get_markets_region_id_orders(order_type="all", type_id=sde_type, region_id=region_id)) - - -@app.get("/_tools/get_all_mat") -async def _get_all_mat(db: Session = Depends(get_session)): - materials = db.query(models_sde.SDETypeMaterial).filter(models_sde.SDETypeMaterial.type.has(models_sde.SDEType.published == True)).all() - allmat = set() - for mat in materials: - allmat.add(mat.material_type.name) - return allmat - - -if __name__ == "__main__": - import uvicorn - uvicorn.run("eveal.main:app", host="0.0.0.0", port=8000, reload=True) diff --git a/eveal/models_esi.py b/eveal/models_esi.py deleted file mode 100644 index 0b26ed6..0000000 --- a/eveal/models_esi.py +++ /dev/null @@ -1,24 +0,0 @@ -from datetime import datetime -from typing import Optional, List -from sqlmodel import SQLModel, Field, Relationship - - -class ESIMarketOrder(SQLModel, table=True): - id: Optional[int] = Field(default=None, primary_key=True) - timestamp: datetime = Field(default_factory=datetime.utcnow, nullable=False) - - region_id: int - type_id: int # TODO: link to SDE - location_id: int # TODO: link to SDE - volume_total: int - volume_remain: int - min_volume: int - order_id: int # TODO: use this as PK ? (will lose volume_remain history) - price: float - is_buy_order: bool - duration: int - issued: datetime - range: str # TODO: enum? - system_id: int # TODO: link to SDE - - diff --git a/eveal/models_sde.py b/eveal/models_sde.py deleted file mode 100644 index 2d6e0be..0000000 --- a/eveal/models_sde.py +++ /dev/null @@ -1,127 +0,0 @@ -from typing import Optional, List -from sqlmodel import SQLModel, Field, Relationship -from functools import partial - -DynamicRelationship = partial(Relationship, sa_relationship_kwargs={"lazy": "dynamic"}) # change default lazy loading to dynamic - - -class SDEIcon(SQLModel, table=True): - id: Optional[int] = Field(default=None, primary_key=True) - - description: Optional[str] = None - iconFile: str - - categories: List['SDECategory'] = DynamicRelationship(back_populates="icon") - groups: List['SDEGroup'] = DynamicRelationship(back_populates="icon") - marketgroups: List['SDEMarketGroup'] = DynamicRelationship(back_populates="icon") - types: List['SDEType'] = DynamicRelationship(back_populates="icon") - metagroups: List['SDEMetaGroup'] = DynamicRelationship(back_populates="icon") - - -class SDECategory(SQLModel, table=True): - id: Optional[int] = Field(default=None, primary_key=True) - - icon_id: Optional[int] = Field(default=None, foreign_key="sdeicon.id") - icon: Optional[SDEIcon] = Relationship(back_populates="categories") - - name: str - published: bool - - groups: List['SDEGroup'] = DynamicRelationship(back_populates="category") - - -class SDEGroup(SQLModel, table=True): - id: Optional[int] = Field(default=None, primary_key=True) - - anchorable: bool - anchored: bool - - category_id: Optional[int] = Field(default=None, foreign_key="sdecategory.id") - category: Optional[SDECategory] = Relationship(back_populates="groups") - - fittableNonSingletion: bool - - icon_id: Optional[int] = Field(default=None, foreign_key="sdeicon.id") - icon: Optional[SDEIcon] = Relationship(back_populates="groups") - - name: str - published: bool - useBasePrice: bool - - types: List['SDEType'] = DynamicRelationship(back_populates="group") - - -class SDEMarketGroup(SQLModel, table=True): - id: Optional[int] = Field(default=None, primary_key=True) - - description: Optional[str] = None - hasTypes: bool - - icon_id: Optional[int] = Field(default=None, foreign_key="sdeicon.id") - icon: Optional[SDEIcon] = Relationship(back_populates="marketgroups") - - name: str - - parent_marketgroup_id: Optional[int] = Field(default=None, foreign_key="sdemarketgroup.id") - parent_marketgroup: Optional['SDEMarketGroup'] = Relationship(back_populates="children_marketgroups", - sa_relationship_kwargs={"remote_side": 'SDEMarketGroup.id'}) # workaround for self reference: https://github.com/tiangolo/sqlmodel/issues/127#issuecomment-1224135123 - children_marketgroups: List['SDEMarketGroup'] = DynamicRelationship(back_populates="parent_marketgroup") - - types: List['SDEType'] = DynamicRelationship(back_populates="marketgroup") - - -class SDEMetaGroup(SQLModel, table=True): - id: int = Field(primary_key=True) - - name: str - iconSuffix: Optional[str] = None - - icon_id: Optional[int] = Field(default=None, foreign_key="sdeicon.id") - icon: Optional[SDEIcon] = Relationship(back_populates="metagroups") - - types: List['SDEType'] = DynamicRelationship(back_populates="metagroup") - - -class SDEType(SQLModel, table=True): - id: int = Field(primary_key=True) - - group_id: Optional[int] = Field(default=None, foreign_key="sdegroup.id") - group: Optional[SDEGroup] = Relationship(back_populates="types") - - marketgroup_id: Optional[int] = Field(default=None, foreign_key="sdemarketgroup.id") - marketgroup: Optional[SDEMarketGroup] = Relationship(back_populates="types") - - metagroup_id: Optional[int] = Field(default=None, foreign_key="sdemetagroup.id") - metagroup: Optional[SDEMetaGroup] = Relationship(back_populates="types") - - name: str - published: bool = False - description: Optional[str] = None - basePrice: Optional[float] = None - - icon_id: Optional[int] = Field(default=None, foreign_key="sdeicon.id") - icon: Optional[SDEIcon] = Relationship(back_populates="types") - - volume: Optional[float] = None - portionSize: int - - materials: List['SDETypeMaterial'] = DynamicRelationship(back_populates="type", - sa_relationship_kwargs={"lazy": "dynamic", "foreign_keys": '[SDETypeMaterial.type_id]'}) # https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1537445078 - material_of: List['SDETypeMaterial'] = DynamicRelationship(back_populates="material_type", - sa_relationship_kwargs={"lazy": "dynamic", "foreign_keys": '[SDETypeMaterial.material_type_id]'}) # https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1537445078 - - -class SDETypeMaterial(SQLModel, table=True): - id: Optional[int] = Field(default=None, primary_key=True) - - type_id: Optional[int] = Field(default=None, foreign_key="sdetype.id") - type: Optional[SDEType] = Relationship(back_populates="materials", - sa_relationship_kwargs={"primaryjoin": 'SDETypeMaterial.type_id==SDEType.id', - 'lazy': 'joined'}) # workaround: https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1002835506 - - material_type_id: Optional[int] = Field(default=None, foreign_key="sdetype.id") - material_type: Optional[SDEType] = Relationship(back_populates="material_of", - sa_relationship_kwargs={"primaryjoin": 'SDETypeMaterial.material_type_id==SDEType.id', - 'lazy': 'joined'}) # workaround: https://github.com/tiangolo/sqlmodel/issues/10#issuecomment-1002835506 - - quantity: int diff --git a/eveal/schemas.py b/eveal/schemas.py deleted file mode 100644 index 5e0d287..0000000 --- a/eveal/schemas.py +++ /dev/null @@ -1,52 +0,0 @@ -from pydantic import BaseModel - - -class Price(BaseModel): - avg: float - max: float - median: float - min: float - percentile: float - stddev: float - volume: int - order_count: int - -class Prices(BaseModel): - all: Price - buy: Price - sell: Price - - -class Item(BaseModel): - name: str - typeID: int - typeName: str - typeVolume: float - quantity: int - prices: Prices - -class Total_price(BaseModel): - buy: float - sell: float - volume: float - -class Evepraisal(BaseModel): - id: str - created: int - kind: str - market_name: str - items: list[Item] - totals: Total_price - price_percentage: float - raw: str - - -class PriceReprocess(BaseModel): - typeID: int - buy: float - sell: float - buy_reprocess: float - sell_reprocess: float - name: str - - diff --git a/import_sde.py b/import_sde.py deleted file mode 100644 index a827335..0000000 --- a/import_sde.py +++ /dev/null @@ -1,166 +0,0 @@ - - -print("Importing SDE data...") -print("Importing icons...") -with open("static_eve/sde/fsd/iconIDs.yaml", "r", encoding="utf-8") as f: - icons = yaml.safe_load(f) -new_icons = total_icons = 0 -with Session(engine) as db: - for id, icon in icons.items(): - db_icon = db.get(models_sde.SDEIcon, id) - if not db_icon: - db_icon = models_sde.SDEIcon(id=id) - db.add(db_icon) - new_icons += 1 - db_icon.iconFile = icon['iconFile'] - db_icon.description = icon['description'] if 'description' in icon else None - total_icons += 1 - db.commit() -print(f"Imported {new_icons} new icons / {total_icons} total icons.") - - -print("Importing categories...") -with open("static_eve/sde/fsd/categoryIDs.yaml", "r", encoding="utf-8") as f: - categories = yaml.safe_load(f) -new_categories = total_categories = 0 -with Session(engine) as db: - for id, category in categories.items(): - # if category["published"] == False: - # continue - db_category = db.get(models_sde.SDECategory, id) - if not db_category: - db_category = models_sde.SDECategory(id=id) - db.add(db_category) - new_categories += 1 - db_category.name = category['name']['en'] - db_category.published = category['published'] - db_category.icon_id = category['iconID'] if 'iconID' in category else None - total_categories += 1 - db.commit() -print(f"Imported {new_categories} new categories / {total_categories} total categories.") - - -print("Importing groups...") -with open("static_eve/sde/fsd/groupIDs.yaml", "r", encoding="utf-8") as f: - groups = yaml.safe_load(f) -new_groups = total_groups = 0 -with Session(engine) as db: - for id, group in groups.items(): - # if group["published"] == False: - # continue - db_group = db.get(models_sde.SDEGroup, id) - if not db_group: - db_group = models_sde.SDEGroup(id=id) - db.add(db_group) - new_groups += 1 - db_group.anchorable = group['anchorable'] - db_group.anchored = group['anchored'] - db_group.category_id = group['categoryID'] - db_group.fittableNonSingletion = group['fittableNonSingleton'] - db_group.icon_id = group['iconID'] if 'iconID' in group else None - db_group.name = group['name']['en'] - db_group.published = group['published'] - db_group.useBasePrice = group['useBasePrice'] - total_groups += 1 - db.commit() -print(f"Imported {new_groups} new groups / {total_groups} total groups.") - - -print("Importing marketgroups...") -with open("static_eve/sde/fsd/marketGroups.yaml", "r", encoding="utf-8") as f: - marketgroups = yaml.safe_load(f) -new_marketgroups = total_marketgroups = 0 -with Session(engine) as db: - for id, marketgroup in marketgroups.items(): - db_marketgroups = db.get(models_sde.SDEMarketGroup, id) - if not db_marketgroups: - db_marketgroups = models_sde.SDEMarketGroup(id=id) - db.add(db_marketgroups) - new_marketgroups += 1 - db_marketgroups.description = marketgroup['descriptionID']['en'] if 'descriptionID' in marketgroup else None - db_marketgroups.hasTypes = marketgroup['hasTypes'] - db_marketgroups.icon_id = marketgroup['iconID'] if 'iconID' in marketgroup else None - db_marketgroups.name = marketgroup['nameID']['en'] - # db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None - total_marketgroups += 1 - db.commit() -print(f"Imported {new_marketgroups} marketgroups / {total_marketgroups} total marketgroups.") -print("Setting up marketgroup Parents...") -total_marketgroup_links = 0 -with Session(engine) as db: - for id, marketgroup in marketgroups.items(): - db_marketgroups = db.get(models_sde.SDEMarketGroup, id) - db_marketgroups.parent_marketgroup_id = marketgroup['parentGroupID'] if 'parentGroupID' in marketgroup else None - total_marketgroup_links += 1 - db.commit() -print(f"Updated {total_marketgroup_links} marketgroup Parents") - - -print("Importing metagroups...") -with open("static_eve/sde/fsd/metaGroups.yaml", "r", encoding="utf-8") as f: - metagroups = yaml.safe_load(f) -new_metagroups = total_metagroups = 0 -with Session(engine) as db: - for id, metagroup in metagroups.items(): - db_metagroups = db.get(models_sde.SDEMetaGroup, id) - if not db_metagroups: - db_metagroups = models_sde.SDEMetaGroup(id=id) - db.add(db_metagroups) - new_metagroups += 1 - db_metagroups.name = metagroup['nameID']['en'] - db_metagroups.iconSuffix = metagroup['iconSuffix'] if 'iconSuffix' in metagroup else None - db_metagroups.icon_id = metagroup['iconID'] if 'iconID' in metagroup else None - total_metagroups += 1 - db.commit() -print(f"Imported {new_metagroups} metagroups / {total_metagroups} total metagroups.") - - -print("Importing types...") -with open("static_eve/sde/fsd/typeIDs.yaml", "r", encoding="utf-8") as f: - types = yaml.safe_load(f) -new_types = total_types = 0 -with Session(engine) as db: - for id, type in types.items(): - # if type["published"] == False: - # continue - db_type = db.get(models_sde.SDEType, id) - if not db_type: - db_type = models_sde.SDEType(id=id) - db.add(db_type) - new_types += 1 - db_type.group_id = type['groupID'] - db_type.marketgroup_id = type['marketGroupID'] if 'marketGroupID' in type else None - db_type.name = type['name']['en'] - db_type.published = type['published'] - db_type.basePrice = type['basePrice'] if 'basePrice' in type else None - db_type.description = type['description']['en'] if 'description' in type else None - db_type.icon_id = type['iconID'] if 'iconID' in type else None - db_type.portionSize = type['portionSize'] - db_type.volume = type['volume'] if 'volume' in type else None - db_type.metagroup_id = type['metaGroupID'] if 'metaGroupID' in type else None - total_types += 1 - db.commit() -print(f"Imported {new_types} types / {total_types} total types.") - - -print("Importing materials...") -with open("static_eve/sde/fsd/typeMaterials.yaml", "r", encoding="utf-8") as f: - materials = yaml.safe_load(f) -new_typemat = total_typemat = 0 -with Session(engine) as db: - for id, material in materials.items(): - for mat in material['materials']: - db_typemat = db.query(models_sde.SDETypeMaterial).filter(models_sde.SDETypeMaterial.type_id == id, models_sde.SDETypeMaterial.material_type_id == mat['materialTypeID']).one_or_none() - if not db_typemat: - db_typemat = models_sde.SDETypeMaterial() - db.add(db_typemat) - new_typemat += 1 - db_typemat.type_id = id - db_typemat.material_type_id = mat['materialTypeID'] - db_typemat.quantity = mat['quantity'] - total_typemat += 1 - db.commit() -print(f"Imported {new_typemat} materials / {total_typemat} total materials.") - - -print("DONE!") \ No newline at end of file -- 2.49.1 From c3002d201fadca0d9538a44179d67b3adc1a3537 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Thu, 26 Oct 2023 20:58:46 +0200 Subject: [PATCH 03/13] new import_sde --- api/management/__init__.py | 0 api/management/commands/__init__.py | 0 api/management/commands/import_sde.py | 140 ++++++++++++++++++++++++++ api/migrations/0001_initial.py | 10 +- api/models_sde.py | 8 +- docker-compose.dev.yml | 3 +- requirements.txt | 2 + 7 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 api/management/__init__.py create mode 100644 api/management/commands/__init__.py create mode 100644 api/management/commands/import_sde.py diff --git a/api/management/__init__.py b/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/__init__.py b/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/import_sde.py b/api/management/commands/import_sde.py new file mode 100644 index 0000000..1fecf3b --- /dev/null +++ b/api/management/commands/import_sde.py @@ -0,0 +1,140 @@ +from django.core.management.base import BaseCommand, CommandError +from api.models import * +import yaml + +class Command(BaseCommand): + help = 'Import SDE data from YAML files' + + def add_arguments(self, parser): + parser.add_argument('path', type=str, help='Path to the SDE YAML files') + + def handle(self, *args, **options): + # Import the SDE data + self.stdout.write(self.style.SUCCESS('Importing SDE data')) + self.import_sde_data(path=options['path']) + self.stdout.write(self.style.SUCCESS('Successfully imported SDE data')) + + def import_sde_data(self, path): + self.import_icons(path + "/iconIDs.yaml") + self.import_categories(path + "/categoryIDs.yaml") + self.import_groups(path + "/groupIDs.yaml") + self.import_marketgroups(path + "/marketGroups.yaml") + self.import_metagroups(path + "/metaGroups.yaml") + self.import_types(path + "/typeIDs.yaml") + self.import_type_materials(path + "/typeMaterials.yaml") + + def import_icons(self, path): + with open(path) as file: + icons = yaml.load(file, Loader=yaml.FullLoader) + for id, icon in icons.items(): + SDEIcon.objects.update_or_create( + id=id, + defaults={ + 'iconFile': icon['iconFile'], + 'description': icon['description'] if "description" in icon else "" + } + ) + self.stdout.write(self.style.SUCCESS('Icons imported')) + + def import_categories(self, path): + with open(path) as file: + categories = yaml.load(file, Loader=yaml.FullLoader) + for id, category in categories.items(): + SDECategory.objects.update_or_create( + id=id, + defaults={ + 'icon': SDEIcon.objects.get(id=category['iconID']) if "iconID" in category else None, + 'name': category['name']['en'], + 'published': category['published'] + } + ) + self.stdout.write(self.style.SUCCESS('Categories imported')) + + def import_groups(self, path): + with open(path) as file: + groups = yaml.load(file, Loader=yaml.FullLoader) + for id, group in groups.items(): + SDEGroup.objects.update_or_create( + id=id, + defaults={ + 'category': SDECategory.objects.get(id=group['categoryID']), + 'name': group['name'], + 'published': group['published'], + 'useBasePrice': group['useBasePrice'], + 'fittableNonSingleton': group['fittableNonSingleton'], + 'anchored': group['anchored'], + 'anchorable': group['anchorable'], + 'icon': SDEIcon.objects.get(id=group['iconID']) if "iconID" in group else None + } + ) + self.stdout.write(self.style.SUCCESS('Groups imported')) + + def import_marketgroups(self, path): + with open(path) as file: + marketgroups = yaml.load(file, Loader=yaml.FullLoader) + for id, marketgroup in marketgroups.items(): + SDEMarektGroup.objects.update_or_create( + id=id, + defaults={ + 'icon': SDEIcon.objects.get(id=marketgroup['iconID']) if "iconID" in marketgroup else None, + 'name': marketgroup['nameID']['en'], + 'description': marketgroup['descriptionID']['en'] if "descriptionID" in marketgroup else "", + 'hasTypes': marketgroup['hasTypes'], + } + ) + self.stdout.write(self.style.SUCCESS('Marketgroups imported')) + for id, marketgroup in marketgroups.items(): + if "parentGroupID" in marketgroup: + sde_mg = SDEMarektGroup.objects.get(id=id) + sde_mg.parent_marketgroup = SDEMarektGroup.objects.get(id=marketgroup['parentGroupID']) + sde_mg.save() + self.stdout.write(self.style.SUCCESS('Marketgroups linked')) + + def import_metagroups(self, path): + with open(path) as file: + metagroups = yaml.load(file, Loader=yaml.FullLoader) + for id, metagroup in metagroups.items(): + SDEMetaGroup.objects.update_or_create( + id=id, + defaults={ + 'icon': SDEIcon.objects.get(id=metagroup['iconID']) if "iconID" in metagroup else None, + 'name': metagroup['nameID']['en'], + 'iconSuffix': metagroup['iconSuffix'] if "iconSuffix" in metagroup else "", + } + ) + self.stdout.write(self.style.SUCCESS('Metagroups imported')) + + def import_types(self, path): + with open(path) as file: + types = yaml.load(file, Loader=yaml.FullLoader) + for id, type in types.items(): + SDEType.objects.update_or_create( + id=id, + defaults={ + 'group': SDEGroup.objects.get(id=type['groupID']), + 'marketgroup': SDEMarektGroup.objects.get(id=type['marketGroupID']) if "marketGroupID" in type else None, + 'metagroup': SDEMetaGroup.objects.get(id=type['metaGroupID']) if "metaGroupID" in type else None, + 'name': type['name']['en'], + 'description': type['description']['en'] if "description" in type else "", + 'published': type['published'], + 'basePrice': type['basePrice'] if "basePrice" in type else 0, + 'icon': SDEIcon.objects.get(id=type['iconID']) if "iconID" in type else None, + 'volume': type['volume'] if "volume" in type else 0, + 'portionSize': type['portionSize'], + } + ) + self.stdout.write(self.style.SUCCESS('Types imported')) + + def import_type_materials(self, path): + with open(path) as file: + type_materials = yaml.load(file, Loader=yaml.FullLoader) + for id, type_material in type_materials.items(): + for material in type_material['materials']: + SDETypeMaterial.objects.update_or_create( + type=SDEType.objects.get(id=id), + material_type=SDEType.objects.get(id=material['materialTypeID']), + defaults={ + 'quantity': material['quantity'], + } + ) + self.stdout.write(self.style.SUCCESS('Materials imported')) diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py index 77f2d3a..4297bfd 100644 --- a/api/migrations/0001_initial.py +++ b/api/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.6 on 2023-10-26 15:51 +# Generated by Django 4.2.6 on 2023-10-26 18:34 from django.db import migrations, models import django.db.models.deletion @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('name', models.CharField()), ('published', models.BooleanField()), ('useBasePrice', models.BooleanField()), - ('fittableNonSingletion', models.BooleanField()), + ('fittableNonSingleton', models.BooleanField()), ('anchored', models.BooleanField()), ('anchorable', models.BooleanField()), ('category', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='api.sdecategory')), @@ -73,13 +73,13 @@ class Migration(migrations.Migration): ('portionSize', models.IntegerField()), ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdegroup')), ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdeicon')), - ('marketgroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdemarektgroup')), + ('marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdemarektgroup')), ], ), migrations.CreateModel( name='SDETypeMaterial', fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('quantity', models.IntegerField()), ('material_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials_of', to='api.sdetype')), ('type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials', to='api.sdetype')), @@ -93,7 +93,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='sdetype', name='metagroup', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdemetagroup'), + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdemetagroup'), ), migrations.AddField( model_name='sdegroup', diff --git a/api/models_sde.py b/api/models_sde.py index 8b3ff62..2b38ec7 100644 --- a/api/models_sde.py +++ b/api/models_sde.py @@ -20,7 +20,7 @@ class SDEGroup(models.Model): name = models.CharField() published = models.BooleanField() useBasePrice = models.BooleanField() - fittableNonSingletion = models.BooleanField() + fittableNonSingleton = models.BooleanField() anchored = models.BooleanField() anchorable = models.BooleanField() icon = models.ForeignKey(SDEIcon, related_name="groups", null=True, on_delete=models.SET_NULL) @@ -45,8 +45,8 @@ class SDEMetaGroup(models.Model): class SDEType(models.Model): id = models.IntegerField(primary_key=True) group = models.ForeignKey(SDEGroup, related_name="types", on_delete=models.CASCADE) - marketgroup = models.ForeignKey(SDEMarektGroup, related_name="types", on_delete=models.CASCADE) - metagroup = models.ForeignKey(SDEMetaGroup, related_name="types", on_delete=models.CASCADE) + marketgroup = models.ForeignKey(SDEMarektGroup, related_name="types", on_delete=models.SET_NULL, null=True) + metagroup = models.ForeignKey(SDEMetaGroup, related_name="types", on_delete=models.SET_NULL, null=True) name = models.CharField() description = models.CharField() published = models.BooleanField() @@ -58,9 +58,7 @@ class SDEType(models.Model): class SDETypeMaterial(models.Model): - id = models.IntegerField(primary_key=True) type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials") material_type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials_of") quantity = models.IntegerField() - diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 46800e9..90c594f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -12,7 +12,7 @@ services: - ./mabras:/app/mabras - ./api:/app/api - ./manage.py:/app/manage.py - command: "python manage.py makemigrations;python manage.py migrate" + command: sh -c "python manage.py makemigrations && python manage.py migrate" depends_on: db: condition: service_healthy @@ -28,6 +28,7 @@ services: - ./mabras:/app/mabras - ./api:/app/api - ./manage.py:/app/manage.py + - ./static_eve:/app/static_eve command: ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000", "--reload"] depends_on: redis: diff --git a/requirements.txt b/requirements.txt index 32efb56..b87e5c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ django-filter markdown uvicorn[standard] psycopg[binary] +redis +esy -- 2.49.1 From 24a1cd051a06d1de7e1a43046d1c2ade4972457f Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Thu, 26 Oct 2023 22:14:13 +0200 Subject: [PATCH 04/13] expose sde --- api/serializers_sde.py | 52 ++++++++++++++++++++++++++++++++++++++++++ api/views_sde.py | 47 ++++++++++++++++++++++++++++++++++++++ {api => mabras}/esi.py | 0 mabras/urls.py | 9 +++++++- 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 api/serializers_sde.py create mode 100644 api/views_sde.py rename {api => mabras}/esi.py (100%) diff --git a/api/serializers_sde.py b/api/serializers_sde.py new file mode 100644 index 0000000..cef88e0 --- /dev/null +++ b/api/serializers_sde.py @@ -0,0 +1,52 @@ +from api.models_sde import * +from rest_framework import serializers + + +class SDEIconSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDEIcon + # fields = ['id', 'iconFile', 'description'] + fields = "__all__" + + +class SDECategorySerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDECategory + # fields = ['id', 'icon', 'name', 'published'] + fields = "__all__" + + +class SDEGroupSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDEGroup + # fields = ['id', 'category', 'name', 'published', 'useBasePrice', 'fittableNonSingleton', 'anchored', 'anchorable', 'icon'] + fields = "__all__" + + +class SDEMarektGroupSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDEMarektGroup + # fields = ['id', 'icon', 'name', 'description', 'hasTypes', 'parent_marketgroup'] + fields = "__all__" + + +class SDEMetaGroupSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDEMetaGroup + # fields = ['id', 'icon', 'name', 'iconSuffix'] + fields = "__all__" + + +class SDETypeSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDEType + # fields = ['id', 'group', 'marketgroup', 'metagroup', 'name', 'description', 'published', 'basePrice', 'icon', 'volume', 'portionSize', 'materials'] + fields = "__all__" + + +class SDETypeMaterialSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = SDETypeMaterial + # fields = ['type', 'material', 'quantity'] + fields = "__all__" + diff --git a/api/views_sde.py b/api/views_sde.py new file mode 100644 index 0000000..64654e7 --- /dev/null +++ b/api/views_sde.py @@ -0,0 +1,47 @@ +from api.models_sde import * +from api.serializers_sde import * +from rest_framework import viewsets + + +class SDEIconViewSet(viewsets.ModelViewSet): + queryset = SDEIcon.objects.all() + serializer_class = SDEIconSerializer + # permission_classes = [permissions.IsAuthenticated] + + +class SDECategoryViewSet(viewsets.ModelViewSet): + queryset = SDECategory.objects.all() + serializer_class = SDECategorySerializer + # permission_classes = [permissions.IsAuthenticated] + + +class SDEGroupViewSet(viewsets.ModelViewSet): + queryset = SDEGroup.objects.all() + serializer_class = SDEGroupSerializer + # permission_classes = [permissions.IsAuthenticated] + + +class SDEMarketGroupViewSet(viewsets.ModelViewSet): + queryset = SDEMarektGroup.objects.all() + serializer_class = SDEMarektGroupSerializer + # permission_classes = [permissions.IsAuthenticated] + + +class SDEMetaGroupViewSet(viewsets.ModelViewSet): + queryset = SDEMetaGroup.objects.all() + serializer_class = SDEMetaGroupSerializer + # permission_classes = [permissions.IsAuthenticated] + + +class SDETypeViewSet(viewsets.ModelViewSet): + queryset = SDEType.objects.all() + serializer_class = SDETypeSerializer + # permission_classes = [permissions.IsAuthenticated] + + +class SDETypeMaterialViewSet(viewsets.ModelViewSet): + queryset = SDETypeMaterial.objects.all() + serializer_class = SDETypeMaterialSerializer + # permission_classes = [permissions.IsAuthenticated] + + diff --git a/api/esi.py b/mabras/esi.py similarity index 100% rename from api/esi.py rename to mabras/esi.py diff --git a/mabras/urls.py b/mabras/urls.py index 3d54102..38db0c3 100644 --- a/mabras/urls.py +++ b/mabras/urls.py @@ -17,11 +17,18 @@ Including another URLconf from django.contrib import admin from django.urls import include, path from rest_framework import routers -from api import views +from api import views, views_sde router = routers.DefaultRouter() router.register(r'users', views.UserViewSet) router.register(r'groups', views.GroupViewSet) +router.register(r'sde/icons', views_sde.SDEIconViewSet) +router.register(r'sde/categories', views_sde.SDECategoryViewSet) +router.register(r'sde/groups', views_sde.SDEGroupViewSet) +router.register(r'sde/marketgroups', views_sde.SDEMarketGroupViewSet) +router.register(r'sde/metagroups', views_sde.SDEMetaGroupViewSet) +router.register(r'sde/types', views_sde.SDETypeViewSet) +router.register(r'sde/typematerials', views_sde.SDETypeMaterialViewSet) # Wire up our API using automatic URL routing. # Additionally, we include login URLs for the browsable API. -- 2.49.1 From 3615689704a817ce478dbb4c5318141b60d21510 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sat, 28 Oct 2023 13:32:04 +0200 Subject: [PATCH 05/13] independant sde app --- api/migrations/0001_initial.py | 108 ------------------ api/models.py | 4 +- api/models_esi.py | 2 - api/templates/redoc.html | 21 ++++ api/templates/swagger-ui.html | 28 +++++ api/urls.py | 7 ++ docker-compose.dev.yml | 4 +- mabras/settings.py | 1 + mabras/urls.py | 37 +++--- requirements.txt | 2 + {api/management => sde}/__init__.py | 0 sde/apps.py | 6 + .../commands => sde/management}/__init__.py | 0 sde/management/commands/__init__.py | 0 .../management/commands/import_sde.py | 2 +- sde/migrations/__init__.py | 0 api/models_sde.py => sde/models.py | 0 api/serializers_sde.py => sde/serializers.py | 30 ++--- sde/urls.py | 12 ++ api/views_sde.py => sde/views.py | 17 ++- 20 files changed, 127 insertions(+), 154 deletions(-) delete mode 100644 api/migrations/0001_initial.py delete mode 100644 api/models_esi.py create mode 100644 api/templates/redoc.html create mode 100644 api/templates/swagger-ui.html create mode 100644 api/urls.py rename {api/management => sde}/__init__.py (100%) create mode 100644 sde/apps.py rename {api/management/commands => sde/management}/__init__.py (100%) create mode 100644 sde/management/commands/__init__.py rename {api => sde}/management/commands/import_sde.py (99%) create mode 100644 sde/migrations/__init__.py rename api/models_sde.py => sde/models.py (100%) rename api/serializers_sde.py => sde/serializers.py (66%) create mode 100644 sde/urls.py rename api/views_sde.py => sde/views.py (71%) diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py deleted file mode 100644 index 4297bfd..0000000 --- a/api/migrations/0001_initial.py +++ /dev/null @@ -1,108 +0,0 @@ -# Generated by Django 4.2.6 on 2023-10-26 18:34 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='SDECategory', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('name', models.CharField()), - ('published', models.BooleanField()), - ], - ), - migrations.CreateModel( - name='SDEGroup', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('name', models.CharField()), - ('published', models.BooleanField()), - ('useBasePrice', models.BooleanField()), - ('fittableNonSingleton', models.BooleanField()), - ('anchored', models.BooleanField()), - ('anchorable', models.BooleanField()), - ('category', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='api.sdecategory')), - ], - ), - migrations.CreateModel( - name='SDEIcon', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('iconFile', models.CharField()), - ('description', models.CharField()), - ], - ), - migrations.CreateModel( - name='SDEMarektGroup', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('name', models.CharField()), - ('description', models.CharField(default='')), - ('hasTypes', models.BooleanField()), - ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.sdeicon')), - ('parent_marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='child_marketgroups', to='api.sdemarektgroup')), - ], - ), - migrations.CreateModel( - name='SDEMetaGroup', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('name', models.CharField()), - ('iconSuffix', models.CharField()), - ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='metagroups', to='api.sdeicon')), - ], - ), - migrations.CreateModel( - name='SDEType', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('name', models.CharField()), - ('description', models.CharField()), - ('published', models.BooleanField()), - ('basePrice', models.FloatField()), - ('volume', models.FloatField()), - ('portionSize', models.IntegerField()), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='api.sdegroup')), - ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdeicon')), - ('marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdemarektgroup')), - ], - ), - migrations.CreateModel( - name='SDETypeMaterial', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('quantity', models.IntegerField()), - ('material_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials_of', to='api.sdetype')), - ('type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials', to='api.sdetype')), - ], - ), - migrations.AddField( - model_name='sdetype', - name='materials', - field=models.ManyToManyField(related_name='material_of', through='api.SDETypeMaterial', to='api.sdetype'), - ), - migrations.AddField( - model_name='sdetype', - name='metagroup', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='api.sdemetagroup'), - ), - migrations.AddField( - model_name='sdegroup', - name='icon', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='groups', to='api.sdeicon'), - ), - migrations.AddField( - model_name='sdecategory', - name='icon', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='categories', to='api.sdeicon'), - ), - ] diff --git a/api/models.py b/api/models.py index fa6a91e..d60611a 100644 --- a/api/models.py +++ b/api/models.py @@ -1,4 +1,2 @@ -from django.db import models -from .models_esi import * -from .models_sde import * +# models \ No newline at end of file diff --git a/api/models_esi.py b/api/models_esi.py deleted file mode 100644 index beeb308..0000000 --- a/api/models_esi.py +++ /dev/null @@ -1,2 +0,0 @@ -from django.db import models - diff --git a/api/templates/redoc.html b/api/templates/redoc.html new file mode 100644 index 0000000..c02bbf9 --- /dev/null +++ b/api/templates/redoc.html @@ -0,0 +1,21 @@ + + + + ReDoc + + + + + + + + + + + + \ No newline at end of file diff --git a/api/templates/swagger-ui.html b/api/templates/swagger-ui.html new file mode 100644 index 0000000..2977649 --- /dev/null +++ b/api/templates/swagger-ui.html @@ -0,0 +1,28 @@ + + + + Swagger + + + + + +
+ + + + \ No newline at end of file diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 0000000..c8711f0 --- /dev/null +++ b/api/urls.py @@ -0,0 +1,7 @@ +from . import views +from django.urls import include, path +from rest_framework import routers + +router = routers.DefaultRouter() +router.register(r'users', views.UserViewSet) +router.register(r'groups', views.GroupViewSet) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 90c594f..54b52c5 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,6 +11,7 @@ services: volumes: - ./mabras:/app/mabras - ./api:/app/api + - ./sde:/app/sde - ./manage.py:/app/manage.py command: sh -c "python manage.py makemigrations && python manage.py migrate" depends_on: @@ -27,8 +28,9 @@ services: volumes: - ./mabras:/app/mabras - ./api:/app/api + - ./sde:/app/sde - ./manage.py:/app/manage.py - - ./static_eve:/app/static_eve +# - ./static_eve:/app/static_eve command: ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000", "--reload"] depends_on: redis: diff --git a/mabras/settings.py b/mabras/settings.py index 81b72fd..4437214 100644 --- a/mabras/settings.py +++ b/mabras/settings.py @@ -44,6 +44,7 @@ REST_FRAMEWORK = { INSTALLED_APPS = [ 'api', + 'sde', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', diff --git a/mabras/urls.py b/mabras/urls.py index 38db0c3..e89d4e4 100644 --- a/mabras/urls.py +++ b/mabras/urls.py @@ -14,25 +14,30 @@ Including another URLconf 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -from django.contrib import admin from django.urls import include, path from rest_framework import routers -from api import views, views_sde - -router = routers.DefaultRouter() -router.register(r'users', views.UserViewSet) -router.register(r'groups', views.GroupViewSet) -router.register(r'sde/icons', views_sde.SDEIconViewSet) -router.register(r'sde/categories', views_sde.SDECategoryViewSet) -router.register(r'sde/groups', views_sde.SDEGroupViewSet) -router.register(r'sde/marketgroups', views_sde.SDEMarketGroupViewSet) -router.register(r'sde/metagroups', views_sde.SDEMetaGroupViewSet) -router.register(r'sde/types', views_sde.SDETypeViewSet) -router.register(r'sde/typematerials', views_sde.SDETypeMaterialViewSet) +from rest_framework.schemas import get_schema_view +from django.views.generic import TemplateView +from sde.urls import router as sde_router +from api.urls import router as api_router # Wire up our API using automatic URL routing. # Additionally, we include login URLs for the browsable API. urlpatterns = [ - path('', include(router.urls)), - path('api-auth/', include('rest_framework.urls', namespace='rest_framework')) -] \ No newline at end of file + path('', include(api_router.urls)), + path('sde/', include(sde_router.urls)), + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), + path('openapi/', get_schema_view( + title="Mabras", + description="API for EvEal", + version="0.0.9" + ), name='openapi-schema'), + path('swagger-ui/', TemplateView.as_view( + template_name='swagger-ui.html', + extra_context={'schema_url': 'openapi-schema'} + ), name='swagger-ui'), + path('redoc/', TemplateView.as_view( + template_name='redoc.html', + extra_context={'schema_url': 'openapi-schema'} + ), name='redoc'), +] diff --git a/requirements.txt b/requirements.txt index b87e5c7..97a7883 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,5 @@ uvicorn[standard] psycopg[binary] redis esy +pyyaml +uritemplate diff --git a/api/management/__init__.py b/sde/__init__.py similarity index 100% rename from api/management/__init__.py rename to sde/__init__.py diff --git a/sde/apps.py b/sde/apps.py new file mode 100644 index 0000000..7ab8d8d --- /dev/null +++ b/sde/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class SdeConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'sde' diff --git a/api/management/commands/__init__.py b/sde/management/__init__.py similarity index 100% rename from api/management/commands/__init__.py rename to sde/management/__init__.py diff --git a/sde/management/commands/__init__.py b/sde/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/import_sde.py b/sde/management/commands/import_sde.py similarity index 99% rename from api/management/commands/import_sde.py rename to sde/management/commands/import_sde.py index 1fecf3b..1582bb2 100644 --- a/api/management/commands/import_sde.py +++ b/sde/management/commands/import_sde.py @@ -1,5 +1,5 @@ from django.core.management.base import BaseCommand, CommandError -from api.models import * +from sde.models import * import yaml class Command(BaseCommand): diff --git a/sde/migrations/__init__.py b/sde/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/models_sde.py b/sde/models.py similarity index 100% rename from api/models_sde.py rename to sde/models.py diff --git a/api/serializers_sde.py b/sde/serializers.py similarity index 66% rename from api/serializers_sde.py rename to sde/serializers.py index cef88e0..d0563b7 100644 --- a/api/serializers_sde.py +++ b/sde/serializers.py @@ -1,52 +1,54 @@ -from api.models_sde import * +from sde.models import * from rest_framework import serializers -class SDEIconSerializer(serializers.HyperlinkedModelSerializer): +class SDEIconSerializer(serializers.ModelSerializer): class Meta: model = SDEIcon # fields = ['id', 'iconFile', 'description'] fields = "__all__" -class SDECategorySerializer(serializers.HyperlinkedModelSerializer): +class SDECategorySerializer(serializers.ModelSerializer): class Meta: model = SDECategory # fields = ['id', 'icon', 'name', 'published'] fields = "__all__" -class SDEGroupSerializer(serializers.HyperlinkedModelSerializer): +class SDEGroupSerializer(serializers.ModelSerializer): class Meta: model = SDEGroup # fields = ['id', 'category', 'name', 'published', 'useBasePrice', 'fittableNonSingleton', 'anchored', 'anchorable', 'icon'] fields = "__all__" -class SDEMarektGroupSerializer(serializers.HyperlinkedModelSerializer): +class SDEMarektGroupSerializer(serializers.ModelSerializer): class Meta: model = SDEMarektGroup # fields = ['id', 'icon', 'name', 'description', 'hasTypes', 'parent_marketgroup'] fields = "__all__" -class SDEMetaGroupSerializer(serializers.HyperlinkedModelSerializer): +class SDEMetaGroupSerializer(serializers.ModelSerializer): class Meta: model = SDEMetaGroup # fields = ['id', 'icon', 'name', 'iconSuffix'] fields = "__all__" -class SDETypeSerializer(serializers.HyperlinkedModelSerializer): +class SDETypeMaterialSerializer(serializers.ModelSerializer): + class Meta: + model = SDETypeMaterial + # fields = ['type', 'material', 'quantity'] + # fields = "__all__" + exclude = ['id', 'type'] + + +class SDETypeSerializer(serializers.ModelSerializer): + typematerials = SDETypeMaterialSerializer(many=True, read_only=True) class Meta: model = SDEType # fields = ['id', 'group', 'marketgroup', 'metagroup', 'name', 'description', 'published', 'basePrice', 'icon', 'volume', 'portionSize', 'materials'] fields = "__all__" - -class SDETypeMaterialSerializer(serializers.HyperlinkedModelSerializer): - class Meta: - model = SDETypeMaterial - # fields = ['type', 'material', 'quantity'] - fields = "__all__" - diff --git a/sde/urls.py b/sde/urls.py new file mode 100644 index 0000000..905d28b --- /dev/null +++ b/sde/urls.py @@ -0,0 +1,12 @@ +from . import views +from django.urls import include, path +from rest_framework import routers + +router = routers.DefaultRouter() +router.register(r'icons', views.SDEIconViewSet) +router.register(r'categories', views.SDECategoryViewSet) +router.register(r'groups', views.SDEGroupViewSet) +router.register(r'marketgroups', views.SDEMarketGroupViewSet) +router.register(r'metagroups', views.SDEMetaGroupViewSet) +router.register(r'types', views.SDETypeViewSet) +router.register(r'typematerials', views.SDETypeMaterialViewSet) diff --git a/api/views_sde.py b/sde/views.py similarity index 71% rename from api/views_sde.py rename to sde/views.py index 64654e7..e831353 100644 --- a/api/views_sde.py +++ b/sde/views.py @@ -1,45 +1,44 @@ -from api.models_sde import * -from api.serializers_sde import * +from sde.serializers import * from rest_framework import viewsets -class SDEIconViewSet(viewsets.ModelViewSet): +class SDEIconViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDEIcon.objects.all() serializer_class = SDEIconSerializer # permission_classes = [permissions.IsAuthenticated] -class SDECategoryViewSet(viewsets.ModelViewSet): +class SDECategoryViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDECategory.objects.all() serializer_class = SDECategorySerializer # permission_classes = [permissions.IsAuthenticated] -class SDEGroupViewSet(viewsets.ModelViewSet): +class SDEGroupViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDEGroup.objects.all() serializer_class = SDEGroupSerializer # permission_classes = [permissions.IsAuthenticated] -class SDEMarketGroupViewSet(viewsets.ModelViewSet): +class SDEMarketGroupViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDEMarektGroup.objects.all() serializer_class = SDEMarektGroupSerializer # permission_classes = [permissions.IsAuthenticated] -class SDEMetaGroupViewSet(viewsets.ModelViewSet): +class SDEMetaGroupViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDEMetaGroup.objects.all() serializer_class = SDEMetaGroupSerializer # permission_classes = [permissions.IsAuthenticated] -class SDETypeViewSet(viewsets.ModelViewSet): +class SDETypeViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDEType.objects.all() serializer_class = SDETypeSerializer # permission_classes = [permissions.IsAuthenticated] -class SDETypeMaterialViewSet(viewsets.ModelViewSet): +class SDETypeMaterialViewSet(viewsets.ReadOnlyModelViewSet): queryset = SDETypeMaterial.objects.all() serializer_class = SDETypeMaterialSerializer # permission_classes = [permissions.IsAuthenticated] -- 2.49.1 From 73609e93416698bb64536f7df55ceb0c55eee5bb Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sat, 28 Oct 2023 14:13:14 +0200 Subject: [PATCH 06/13] pep8 --- sde/management/commands/import_sde.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sde/management/commands/import_sde.py b/sde/management/commands/import_sde.py index 1582bb2..2025b7e 100644 --- a/sde/management/commands/import_sde.py +++ b/sde/management/commands/import_sde.py @@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from sde.models import * import yaml + class Command(BaseCommand): help = 'Import SDE data from YAML files' -- 2.49.1 From a1e2f9d8c40fe85cf7ed688dc7ed9dfcd8c9034b Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sat, 28 Oct 2023 14:13:45 +0200 Subject: [PATCH 07/13] initial sde migrations --- sde/migrations/0001_initial.py | 108 +++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 sde/migrations/0001_initial.py diff --git a/sde/migrations/0001_initial.py b/sde/migrations/0001_initial.py new file mode 100644 index 0000000..a76a837 --- /dev/null +++ b/sde/migrations/0001_initial.py @@ -0,0 +1,108 @@ +# Generated by Django 4.2.6 on 2023-10-28 11:26 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SDECategory', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('published', models.BooleanField()), + ], + ), + migrations.CreateModel( + name='SDEGroup', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('published', models.BooleanField()), + ('useBasePrice', models.BooleanField()), + ('fittableNonSingleton', models.BooleanField()), + ('anchored', models.BooleanField()), + ('anchorable', models.BooleanField()), + ('category', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='sde.sdecategory')), + ], + ), + migrations.CreateModel( + name='SDEIcon', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('iconFile', models.CharField()), + ('description', models.CharField()), + ], + ), + migrations.CreateModel( + name='SDEMarektGroup', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('description', models.CharField(default='')), + ('hasTypes', models.BooleanField()), + ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='sde.sdeicon')), + ('parent_marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='child_marketgroups', to='sde.sdemarektgroup')), + ], + ), + migrations.CreateModel( + name='SDEMetaGroup', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('iconSuffix', models.CharField()), + ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='metagroups', to='sde.sdeicon')), + ], + ), + migrations.CreateModel( + name='SDEType', + fields=[ + ('id', models.IntegerField(primary_key=True, serialize=False)), + ('name', models.CharField()), + ('description', models.CharField()), + ('published', models.BooleanField()), + ('basePrice', models.FloatField()), + ('volume', models.FloatField()), + ('portionSize', models.IntegerField()), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='types', to='sde.sdegroup')), + ('icon', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='sde.sdeicon')), + ('marketgroup', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='sde.sdemarektgroup')), + ], + ), + migrations.CreateModel( + name='SDETypeMaterial', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('quantity', models.IntegerField()), + ('material_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials_of', to='sde.sdetype')), + ('type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='typematerials', to='sde.sdetype')), + ], + ), + migrations.AddField( + model_name='sdetype', + name='materials', + field=models.ManyToManyField(related_name='material_of', through='sde.SDETypeMaterial', to='sde.sdetype'), + ), + migrations.AddField( + model_name='sdetype', + name='metagroup', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='types', to='sde.sdemetagroup'), + ), + migrations.AddField( + model_name='sdegroup', + name='icon', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='groups', to='sde.sdeicon'), + ), + migrations.AddField( + model_name='sdecategory', + name='icon', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='categories', to='sde.sdeicon'), + ), + ] -- 2.49.1 From 383d256ea9518780a00075cf31c2c698fada25f6 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sat, 28 Oct 2023 18:34:33 +0200 Subject: [PATCH 08/13] search & eval reprocess --- api/serializers.py | 6 ++-- api/urls.py | 6 ++++ api/views.py | 77 +++++++++++++++++++++++++++++++++++++++++++--- mabras/settings.py | 4 ++- mabras/urls.py | 12 +++----- sde/models.py | 3 ++ sde/serializers.py | 2 +- sde/urls.py | 4 +++ 8 files changed, 97 insertions(+), 17 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index 466c3f6..b6bf275 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1,15 +1,15 @@ from django.contrib.auth.models import User, Group from rest_framework import serializers +from sde.models import SDEType -class UserSerializer(serializers.HyperlinkedModelSerializer): +class UserSerializer(serializers.ModelSerializer): class Meta: model = User fields = ['url', 'username', 'email', 'groups'] -class GroupSerializer(serializers.HyperlinkedModelSerializer): +class GroupSerializer(serializers.ModelSerializer): class Meta: model = Group fields = ['url', 'name'] - diff --git a/api/urls.py b/api/urls.py index c8711f0..cfcc4ef 100644 --- a/api/urls.py +++ b/api/urls.py @@ -5,3 +5,9 @@ from rest_framework import routers router = routers.DefaultRouter() router.register(r'users', views.UserViewSet) router.register(r'groups', views.GroupViewSet) + +urlpatterns = [ + path('', include(router.urls)), + path('types/search', views.custom_types_search, name='custom_types_search'), + path('reprocess/eval', views.reprocess_eval, name='custom_types_search'), +] diff --git a/api/views.py b/api/views.py index 4f59f72..74c0361 100644 --- a/api/views.py +++ b/api/views.py @@ -1,8 +1,14 @@ from django.shortcuts import render from django.contrib.auth.models import User, Group -from rest_framework import viewsets -from rest_framework import permissions -from api.serializers import UserSerializer, GroupSerializer +from rest_framework import viewsets, permissions, settings +from rest_framework.decorators import api_view +from rest_framework.response import Response +from api import serializers +from sde import serializers as sde_serializers +from django.http import JsonResponse +from django.db.models import Q + +from sde.models import SDEType class UserViewSet(viewsets.ModelViewSet): @@ -10,7 +16,7 @@ class UserViewSet(viewsets.ModelViewSet): API endpoint that allows users to be viewed or edited. """ queryset = User.objects.all().order_by('-date_joined') - serializer_class = UserSerializer + serializer_class = serializers.UserSerializer permission_classes = [permissions.IsAuthenticated] @@ -19,6 +25,67 @@ class GroupViewSet(viewsets.ModelViewSet): API endpoint that allows groups to be viewed or edited. """ queryset = Group.objects.all() - serializer_class = GroupSerializer + serializer_class = serializers.GroupSerializer permission_classes = [permissions.IsAuthenticated] + +@api_view(['POST']) +def custom_types_search(request): + items = [] + + for q in request.data: + condition = Q() + for k in q.keys(): + value = q[k] + + token = k.split('___') + key, mods = token[0], token[1:] + + if "i" in mods: + condition = condition & Q(**{key + '__icontains': value}) + elif "in" in mods: + condition = condition & Q(**{key + '__in': value}) + else: + condition = condition & Q(**{key: value}) + + if "not" in mods: + condition = ~condition + + items.extend(SDEType.objects.filter(condition)) + + paginator = settings.api_settings.DEFAULT_PAGINATION_CLASS() + result_page = paginator.paginate_queryset(items, request) + serializer = sde_serializers.SDETypeSerializer(result_page, many=True) + return paginator.get_paginated_response(serializer.data) + + +@api_view(['POST']) +def reprocess_eval(request): + ep_mat = request.data.get("ep_mat") + ep_items = request.data.get("ep_items") + efficiency = request.data.get("efficiency", 0.55) + + matprices = {item["typeID"]: {'sell': item["prices"]["sell"]["min"], 'buy': item["prices"]["buy"]["max"]} for item in ep_mat['items']} + + item_reprocess = [] + for rawitem in ep_items["items"]: + item = SDEType.objects.get(id=rawitem["typeID"]) + buy_reprocess = sell_reprocess = 0.0 + for mat in item.typematerials.all(): + try: + buy_reprocess += matprices[mat.material_type_id]['buy'] * mat.quantity/item.portionSize * efficiency + sell_reprocess += matprices[mat.material_type_id]['sell'] * mat.quantity/item.portionSize * efficiency + except KeyError as e: + JsonResponse({"error": f"No price for material {e}"}) + item_reprocess.append({ + "typeID": item.id, + "name": item.name, + "buy": rawitem["prices"]['buy']["max"], + "sell": rawitem["prices"]['sell']["min"], + "buy_reprocess": buy_reprocess, + "sell_reprocess": sell_reprocess, + }) + + paginator = settings.api_settings.DEFAULT_PAGINATION_CLASS() + result_page = paginator.paginate_queryset(item_reprocess, request) + return paginator.get_paginated_response(result_page) diff --git a/mabras/settings.py b/mabras/settings.py index 4437214..fe1f0f5 100644 --- a/mabras/settings.py +++ b/mabras/settings.py @@ -33,13 +33,14 @@ ALLOWED_HOSTS = [] REST_FRAMEWORK = { 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', - 'PAGE_SIZE': 10, + 'PAGE_SIZE': 25, 'DEFAULT_RENDERER_CLASSES': [ 'rest_framework.renderers.JSONRenderer', ], 'DEFAULT_PARSER_CLASSES': [ 'rest_framework.parsers.JSONParser', ], + 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], } INSTALLED_APPS = [ @@ -51,6 +52,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django_filters', 'rest_framework' ] diff --git a/mabras/urls.py b/mabras/urls.py index e89d4e4..4fe342a 100644 --- a/mabras/urls.py +++ b/mabras/urls.py @@ -15,17 +15,15 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.urls import include, path -from rest_framework import routers from rest_framework.schemas import get_schema_view from django.views.generic import TemplateView -from sde.urls import router as sde_router -from api.urls import router as api_router +from sde.urls import urlpatterns as sde_urls +from api.urls import urlpatterns as api_urls + -# Wire up our API using automatic URL routing. -# Additionally, we include login URLs for the browsable API. urlpatterns = [ - path('', include(api_router.urls)), - path('sde/', include(sde_router.urls)), + path('api/', include(api_urls)), + path('sde/', include(sde_urls)), path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), path('openapi/', get_schema_view( title="Mabras", diff --git a/sde/models.py b/sde/models.py index 2b38ec7..0d0ef09 100644 --- a/sde/models.py +++ b/sde/models.py @@ -56,6 +56,9 @@ class SDEType(models.Model): portionSize = models.IntegerField() materials = models.ManyToManyField("self", through="SDETypeMaterial", symmetrical=False, related_name="material_of") + def __str__(self): + return self.name + class SDETypeMaterial(models.Model): type = models.ForeignKey(SDEType, on_delete=models.CASCADE, related_name="typematerials") diff --git a/sde/serializers.py b/sde/serializers.py index d0563b7..7f57f3f 100644 --- a/sde/serializers.py +++ b/sde/serializers.py @@ -46,7 +46,7 @@ class SDETypeMaterialSerializer(serializers.ModelSerializer): class SDETypeSerializer(serializers.ModelSerializer): - typematerials = SDETypeMaterialSerializer(many=True, read_only=True) + # typematerials = SDETypeMaterialSerializer(many=True, read_only=True) class Meta: model = SDEType # fields = ['id', 'group', 'marketgroup', 'metagroup', 'name', 'description', 'published', 'basePrice', 'icon', 'volume', 'portionSize', 'materials'] diff --git a/sde/urls.py b/sde/urls.py index 905d28b..152956e 100644 --- a/sde/urls.py +++ b/sde/urls.py @@ -10,3 +10,7 @@ router.register(r'marketgroups', views.SDEMarketGroupViewSet) router.register(r'metagroups', views.SDEMetaGroupViewSet) router.register(r'types', views.SDETypeViewSet) router.register(r'typematerials', views.SDETypeMaterialViewSet) + +urlpatterns = [ + path('', include(router.urls)), +] -- 2.49.1 From 33aa805f036a314299010236e80718715503951d Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sat, 28 Oct 2023 18:38:51 +0200 Subject: [PATCH 09/13] fix swagger & redoc templates location --- mabras/settings.py | 4 ++-- {api => mabras}/templates/redoc.html | 0 {api => mabras}/templates/swagger-ui.html | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {api => mabras}/templates/redoc.html (100%) rename {api => mabras}/templates/swagger-ui.html (100%) diff --git a/mabras/settings.py b/mabras/settings.py index fe1f0f5..f785ee9 100644 --- a/mabras/settings.py +++ b/mabras/settings.py @@ -33,7 +33,7 @@ ALLOWED_HOSTS = [] REST_FRAMEWORK = { 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', - 'PAGE_SIZE': 25, + 'PAGE_SIZE': 100, 'DEFAULT_RENDERER_CLASSES': [ 'rest_framework.renderers.JSONRenderer', ], @@ -71,7 +71,7 @@ ROOT_URLCONF = 'mabras.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + 'DIRS': ["mabras/templates"], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ diff --git a/api/templates/redoc.html b/mabras/templates/redoc.html similarity index 100% rename from api/templates/redoc.html rename to mabras/templates/redoc.html diff --git a/api/templates/swagger-ui.html b/mabras/templates/swagger-ui.html similarity index 100% rename from api/templates/swagger-ui.html rename to mabras/templates/swagger-ui.html -- 2.49.1 From f92d518468fe8df5c73a0b25aa3a401e4e0fee74 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sat, 28 Oct 2023 18:49:15 +0200 Subject: [PATCH 10/13] fix dockerfile to add sde app --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e126bfd..4774d88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ COPY --chown=appuser:appuser manage.py /app/manage.py COPY requirements.txt . RUN python -m pip install --no-cache-dir --upgrade -r requirements.txt COPY --chown=appuser:appuser mabras /app/mabras +COPY --chown=appuser:appuser sde /app/sde COPY --chown=appuser:appuser api /app/api USER appuser -- 2.49.1 From f860a5bf1d9024733a95e7de46f58b8e98e52f14 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sun, 29 Oct 2023 12:12:25 +0100 Subject: [PATCH 11/13] add redis cache, Acquisition & django-esi --- .env.template | 1 + api/migrations/0001_initial.py | 31 +++++++++++++++++++++++++++++++ api/models.py | 15 ++++++++++++++- api/serializers.py | 17 ++++++++++++----- api/urls.py | 1 + api/views.py | 28 ++++++++++++++++++---------- mabras/esi.py | 33 --------------------------------- mabras/settings.py | 14 ++++++++++++++ mabras/urls.py | 7 +++---- requirements.txt | 2 +- 10 files changed, 95 insertions(+), 54 deletions(-) create mode 100644 api/migrations/0001_initial.py delete mode 100644 mabras/esi.py diff --git a/.env.template b/.env.template index fb45ed3..428212c 100644 --- a/.env.template +++ b/.env.template @@ -5,6 +5,7 @@ ESI_USER_AGENT= REDIS_URL= REDIS_PORT= +REDIS_DB= REDIS_PASSWORD= SQLITE_DB_PATH= diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py new file mode 100644 index 0000000..976b1fb --- /dev/null +++ b/api/migrations/0001_initial.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.6 on 2023-10-29 11:10 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('sde', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Acquisition', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('quantity', models.IntegerField()), + ('remaining', models.IntegerField()), + ('price', models.FloatField()), + ('date', models.DateTimeField(auto_now_add=True)), + ('source', models.CharField(max_length=255)), + ('type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='acquisitions', to='sde.sdetype')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='acquisitions', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/api/models.py b/api/models.py index d60611a..3041322 100644 --- a/api/models.py +++ b/api/models.py @@ -1,2 +1,15 @@ +from django.db import models +from sde.models import SDEType -# models \ No newline at end of file + +class Acquisition(models.Model): + type = models.ForeignKey(SDEType, related_name="acquisitions", on_delete=models.CASCADE) + + quantity = models.IntegerField() + remaining = models.IntegerField() + + price = models.FloatField() + date = models.DateTimeField(auto_now_add=True) + source = models.CharField(max_length=255) + + user = models.ForeignKey("auth.User", related_name="acquisitions", on_delete=models.CASCADE) diff --git a/api/serializers.py b/api/serializers.py index b6bf275..beebeae 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1,15 +1,22 @@ -from django.contrib.auth.models import User, Group +from django.contrib.auth import models as auth_models from rest_framework import serializers from sde.models import SDEType +from api import models class UserSerializer(serializers.ModelSerializer): class Meta: - model = User - fields = ['url', 'username', 'email', 'groups'] + model = auth_models.User + fields = ['id', 'username', 'email', 'groups'] class GroupSerializer(serializers.ModelSerializer): class Meta: - model = Group - fields = ['url', 'name'] + model = auth_models.Group + fields = ['id', 'name'] + + +class AcquisitionSerializer(serializers.ModelSerializer): + class Meta: + model = models.Acquisition + fields = '__all__' diff --git a/api/urls.py b/api/urls.py index cfcc4ef..0ad437f 100644 --- a/api/urls.py +++ b/api/urls.py @@ -5,6 +5,7 @@ from rest_framework import routers router = routers.DefaultRouter() router.register(r'users', views.UserViewSet) router.register(r'groups', views.GroupViewSet) +router.register(r'acquisitions', views.AcquisitionViewSet) urlpatterns = [ path('', include(router.urls)), diff --git a/api/views.py b/api/views.py index 74c0361..b59b7bb 100644 --- a/api/views.py +++ b/api/views.py @@ -1,21 +1,20 @@ from django.shortcuts import render -from django.contrib.auth.models import User, Group +from django.http import JsonResponse +from django.db.models import Q +from django.contrib.auth import models as auth_models from rest_framework import viewsets, permissions, settings from rest_framework.decorators import api_view from rest_framework.response import Response -from api import serializers -from sde import serializers as sde_serializers -from django.http import JsonResponse -from django.db.models import Q -from sde.models import SDEType +from api import serializers, models +from sde import serializers as sde_serializers, models as sde_models class UserViewSet(viewsets.ModelViewSet): """ API endpoint that allows users to be viewed or edited. """ - queryset = User.objects.all().order_by('-date_joined') + queryset = auth_models.User.objects.all().order_by('-date_joined') serializer_class = serializers.UserSerializer permission_classes = [permissions.IsAuthenticated] @@ -24,11 +23,20 @@ class GroupViewSet(viewsets.ModelViewSet): """ API endpoint that allows groups to be viewed or edited. """ - queryset = Group.objects.all() + queryset = auth_models.Group.objects.all() serializer_class = serializers.GroupSerializer permission_classes = [permissions.IsAuthenticated] +class AcquisitionViewSet(viewsets.ModelViewSet): + """ + API endpoint that allows acquisitions to be viewed or edited. + """ + queryset = models.Acquisition.objects.all().order_by('-date') + serializer_class = serializers.AcquisitionSerializer + permission_classes = [permissions.IsAuthenticated] + + @api_view(['POST']) def custom_types_search(request): items = [] @@ -51,7 +59,7 @@ def custom_types_search(request): if "not" in mods: condition = ~condition - items.extend(SDEType.objects.filter(condition)) + items.extend(sde_models.SDEType.objects.filter(condition)) paginator = settings.api_settings.DEFAULT_PAGINATION_CLASS() result_page = paginator.paginate_queryset(items, request) @@ -69,7 +77,7 @@ def reprocess_eval(request): item_reprocess = [] for rawitem in ep_items["items"]: - item = SDEType.objects.get(id=rawitem["typeID"]) + item = sde_models.SDEType.objects.get(id=rawitem["typeID"]) buy_reprocess = sell_reprocess = 0.0 for mat in item.typematerials.all(): try: diff --git a/mabras/esi.py b/mabras/esi.py deleted file mode 100644 index d186d94..0000000 --- a/mabras/esi.py +++ /dev/null @@ -1,33 +0,0 @@ -import datetime -import os -import redis -import pickle -from esy.client import ESIClient -from esy.auth import ESIAuthenticator - - -class ESICache(object): - def __init__(self, **kwargs): - self._r = redis.Redis(**kwargs) - # self._r = redis.StrictRedis(host=redis_url, port=redis_port, db=db) - - def get(self, key): - # return pickle.loads(self._r[key]) - return pickle.loads(self._r.get(key)) - - def set(self, key, data, cached_until: datetime.datetime): - self._r.set(key, pickle.dumps(data), ex=cached_until - datetime.datetime.now(datetime.timezone.utc)) - - def __contains__(self, item): - # return item in self._r - return self._r.exists(item) - - -esi_client_id = os.getenv('ESI_CLIENT_ID') -esi_secret_key = os.getenv('ESI_SECRET_KEY') - -esi_cache = ESICache(host=os.getenv("REDIS_URL"), port=int(os.getenv("REDIS_PORT")), db="0", - password=os.getenv("REDIS_PASSWD")) - -esi_client = ESIClient.get_client(user_agent=os.getenv('ESI_USER_AGENT'), cache=esi_cache) -esi_auth = ESIAuthenticator() diff --git a/mabras/settings.py b/mabras/settings.py index f785ee9..a002c71 100644 --- a/mabras/settings.py +++ b/mabras/settings.py @@ -46,6 +46,7 @@ REST_FRAMEWORK = { INSTALLED_APPS = [ 'api', 'sde', + 'esi', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -87,6 +88,13 @@ TEMPLATES = [ WSGI_APPLICATION = 'mabras.wsgi.application' +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"redis://{os.getenv('REDIS_URL')}:{os.getenv('REDIS_PORT')}/{os.getenv('REDIS_DB')}", + } +} + # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases @@ -142,3 +150,9 @@ STATIC_URL = 'static/' # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +ESI_SSO_CLIENT_ID = os.getenv("ESI_CLIENT_ID") +ESI_SSO_CLIENT_SECRET = os.getenv("ESI_SECRET_KEY") +ESI_SSO_CALLBACK_URL = os.getenv("ESI_CALLBACK_URL") +ESI_USER_AGENT = os.getenv("ESI_USER_AGENT") +ESI_USER_CONTACT_EMAIL = os.getenv("ESI_USER_AGENT") diff --git a/mabras/urls.py b/mabras/urls.py index 4fe342a..7e158b1 100644 --- a/mabras/urls.py +++ b/mabras/urls.py @@ -17,14 +17,13 @@ Including another URLconf from django.urls import include, path from rest_framework.schemas import get_schema_view from django.views.generic import TemplateView -from sde.urls import urlpatterns as sde_urls -from api.urls import urlpatterns as api_urls urlpatterns = [ - path('api/', include(api_urls)), - path('sde/', include(sde_urls)), + path('api/', include("api.urls")), + path('sde/', include("sde.urls")), path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), + path('sso/', include('esi.urls', namespace='esi')), path('openapi/', get_schema_view( title="Mabras", description="API for EvEal", diff --git a/requirements.txt b/requirements.txt index 97a7883..b304495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ markdown uvicorn[standard] psycopg[binary] redis -esy pyyaml uritemplate +django-esi -- 2.49.1 From ffbf548c7a3dc02d689dfe4a6e6170ba19ec8098 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sun, 29 Oct 2023 14:53:48 +0100 Subject: [PATCH 12/13] fix allowed host env --- .env.template | 3 ++- mabras/settings.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env.template b/.env.template index 428212c..17626bb 100644 --- a/.env.template +++ b/.env.template @@ -15,4 +15,5 @@ POSTGRES_USER= POSTGRES_DB= DRF_SECRET_KEY= -DRF_DEBUG= \ No newline at end of file +DRF_DEBUG= +ALLOWED_HOSTS= diff --git a/mabras/settings.py b/mabras/settings.py index a002c71..6adc5ee 100644 --- a/mabras/settings.py +++ b/mabras/settings.py @@ -11,7 +11,7 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ from pathlib import Path -import os +import os, socket # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -26,8 +26,8 @@ SECRET_KEY = os.getenv("DRF_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.getenv("DRF_DEBUG", False) == "True" -ALLOWED_HOSTS = [] - +ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(",") +ALLOWED_HOSTS = [] if not any(ALLOWED_HOSTS) else ALLOWED_HOSTS # Application definition -- 2.49.1 From 20c5aa9ee58bcec6ac0b7c96d04c83e3c2bf4ba4 Mon Sep 17 00:00:00 2001 From: Kaladaran Date: Sun, 29 Oct 2023 22:01:38 +0100 Subject: [PATCH 13/13] rename mabras -> marbas, fix not condition in search, allow page_size for pagination --- Dockerfile | 4 ++-- api/views.py | 15 ++++++--------- docker-compose.dev.yml | 14 +++++++------- manage.py | 2 +- {mabras => marbas}/__init__.py | 0 {mabras => marbas}/asgi.py | 4 ++-- marbas/pagination.py | 8 ++++++++ {mabras => marbas}/settings.py | 13 ++++++------- {mabras => marbas}/templates/redoc.html | 0 {mabras => marbas}/templates/swagger-ui.html | 0 {mabras => marbas}/urls.py | 4 ++-- {mabras => marbas}/wsgi.py | 4 ++-- 12 files changed, 36 insertions(+), 32 deletions(-) rename {mabras => marbas}/__init__.py (100%) rename {mabras => marbas}/asgi.py (75%) create mode 100644 marbas/pagination.py rename {mabras => marbas}/settings.py (93%) rename {mabras => marbas}/templates/redoc.html (100%) rename {mabras => marbas}/templates/swagger-ui.html (100%) rename {mabras => marbas}/urls.py (95%) rename {mabras => marbas}/wsgi.py (75%) diff --git a/Dockerfile b/Dockerfile index 4774d88..17620d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,9 @@ WORKDIR /app COPY --chown=appuser:appuser manage.py /app/manage.py COPY requirements.txt . RUN python -m pip install --no-cache-dir --upgrade -r requirements.txt -COPY --chown=appuser:appuser mabras /app/mabras +COPY --chown=appuser:appuser marbas /app/marbas COPY --chown=appuser:appuser sde /app/sde COPY --chown=appuser:appuser api /app/api USER appuser -CMD ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "marbas.asgi:application", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/api/views.py b/api/views.py index b59b7bb..7291476 100644 --- a/api/views.py +++ b/api/views.py @@ -42,24 +42,21 @@ def custom_types_search(request): items = [] for q in request.data: - condition = Q() + conditions = Q() for k in q.keys(): value = q[k] token = k.split('___') key, mods = token[0], token[1:] - if "i" in mods: - condition = condition & Q(**{key + '__icontains': value}) - elif "in" in mods: - condition = condition & Q(**{key + '__in': value}) - else: - condition = condition & Q(**{key: value}) + cond = Q(**{key: value}) if "not" in mods: - condition = ~condition + cond = ~cond - items.extend(sde_models.SDEType.objects.filter(condition)) + conditions = conditions & cond + + items.extend(sde_models.SDEType.objects.filter(conditions)) paginator = settings.api_settings.DEFAULT_PAGINATION_CLASS() result_page = paginator.paginate_queryset(items, request) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 54b52c5..2102756 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,7 @@ version: '3' services: migrations: - image: mabras:local + image: marbas:local build: context: . dockerfile: Dockerfile @@ -9,7 +9,7 @@ services: - .env user: "1000:1000" volumes: - - ./mabras:/app/mabras + - ./marbas:/app/marbas - ./api:/app/api - ./sde:/app/sde - ./manage.py:/app/manage.py @@ -19,19 +19,19 @@ services: condition: service_healthy eveal: - image: mabras:local + image: marbas:local env_file: - .env ports: - 8000:8000 user: "1000:1000" volumes: - - ./mabras:/app/mabras + - ./marbas:/app/marbas - ./api:/app/api - ./sde:/app/sde - ./manage.py:/app/manage.py # - ./static_eve:/app/static_eve - command: ["uvicorn", "mabras.asgi:application", "--host", "0.0.0.0", "--port", "8000", "--reload"] + command: ["uvicorn", "marbas.asgi:application", "--host", "0.0.0.0", "--port", "8000", "--reload"] depends_on: redis: condition: service_healthy @@ -55,7 +55,7 @@ services: ports: - 5432:5432 volumes: - - mabras_dbdata:/var/lib/postgresql/data + - marbas_dbdata:/var/lib/postgresql/data env_file: - .env healthcheck: @@ -80,4 +80,4 @@ services: # retries: 3 volumes: - mabras_dbdata: + marbas_dbdata: diff --git a/manage.py b/manage.py index 3a4a74c..54b8e05 100755 --- a/manage.py +++ b/manage.py @@ -6,7 +6,7 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mabras.settings') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marbas.settings') try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/mabras/__init__.py b/marbas/__init__.py similarity index 100% rename from mabras/__init__.py rename to marbas/__init__.py diff --git a/mabras/asgi.py b/marbas/asgi.py similarity index 75% rename from mabras/asgi.py rename to marbas/asgi.py index d714762..82e99c4 100644 --- a/mabras/asgi.py +++ b/marbas/asgi.py @@ -1,5 +1,5 @@ """ -ASGI config for mabras project. +ASGI config for marbas project. It exposes the ASGI callable as a module-level variable named ``application``. @@ -11,6 +11,6 @@ import os from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mabras.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marbas.settings') application = get_asgi_application() diff --git a/marbas/pagination.py b/marbas/pagination.py new file mode 100644 index 0000000..c4d4f9b --- /dev/null +++ b/marbas/pagination.py @@ -0,0 +1,8 @@ +from rest_framework import pagination + + +class CustomPagination(pagination.PageNumberPagination): + page_size = 10 + page_size_query_param = 'page_size' + max_page_size = 250 + diff --git a/mabras/settings.py b/marbas/settings.py similarity index 93% rename from mabras/settings.py rename to marbas/settings.py index 6adc5ee..b5a328f 100644 --- a/mabras/settings.py +++ b/marbas/settings.py @@ -1,5 +1,5 @@ """ -Django settings for mabras project. +Django settings for marbas project. Generated by 'django-admin startproject' using Django 4.2.6. @@ -11,7 +11,7 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ from pathlib import Path -import os, socket +import os # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -32,8 +32,7 @@ ALLOWED_HOSTS = [] if not any(ALLOWED_HOSTS) else ALLOWED_HOSTS # Application definition REST_FRAMEWORK = { - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', - 'PAGE_SIZE': 100, + 'DEFAULT_PAGINATION_CLASS': 'marbas.pagination.CustomPagination', 'DEFAULT_RENDERER_CLASSES': [ 'rest_framework.renderers.JSONRenderer', ], @@ -67,12 +66,12 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'mabras.urls' +ROOT_URLCONF = 'marbas.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': ["mabras/templates"], + 'DIRS': ["marbas/templates"], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -85,7 +84,7 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'mabras.wsgi.application' +WSGI_APPLICATION = 'marbas.wsgi.application' CACHES = { diff --git a/mabras/templates/redoc.html b/marbas/templates/redoc.html similarity index 100% rename from mabras/templates/redoc.html rename to marbas/templates/redoc.html diff --git a/mabras/templates/swagger-ui.html b/marbas/templates/swagger-ui.html similarity index 100% rename from mabras/templates/swagger-ui.html rename to marbas/templates/swagger-ui.html diff --git a/mabras/urls.py b/marbas/urls.py similarity index 95% rename from mabras/urls.py rename to marbas/urls.py index 7e158b1..0c03814 100644 --- a/mabras/urls.py +++ b/marbas/urls.py @@ -1,5 +1,5 @@ """ -URL configuration for mabras project. +URL configuration for marbas project. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/4.2/topics/http/urls/ @@ -25,7 +25,7 @@ urlpatterns = [ path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), path('sso/', include('esi.urls', namespace='esi')), path('openapi/', get_schema_view( - title="Mabras", + title="marbas", description="API for EvEal", version="0.0.9" ), name='openapi-schema'), diff --git a/mabras/wsgi.py b/marbas/wsgi.py similarity index 75% rename from mabras/wsgi.py rename to marbas/wsgi.py index f15aaf8..aab9192 100644 --- a/mabras/wsgi.py +++ b/marbas/wsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for mabras project. +WSGI config for marbas project. It exposes the WSGI callable as a module-level variable named ``application``. @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mabras.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marbas.settings') application = get_wsgi_application() -- 2.49.1