diff --git a/.github/workflows/test-containers.yml b/.github/workflows/test-containers.yml new file mode 100644 index 0000000..88dede5 --- /dev/null +++ b/.github/workflows/test-containers.yml @@ -0,0 +1,19 @@ +name: Test Containers + +on: push + +jobs: + docker: + timeout-minutes: 4 + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Start containers + run: docker-compose -f "docker-compose.yml" up -d --build + + - name: Stop containers + if: always() + run: docker-compose -f "docker-compose.yml" down \ No newline at end of file diff --git a/README.md b/README.md index 375a659..5fb5cd7 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ You can find the code for the mining engine in `oapen-engine/`. * Most popular PostgreSQL database adapter for Python * `pandas` -- data analysis library * Maintained by [PYData](https://pandas.pydata.org/) with large amounts of sponsors. 2,700+ contributors. -* `sklearn` -- Scikit Learn +* `scikit-learn` -- Scikit Learn * Maintained by [a large consortium of corporations and open-source developers](https://scikit-learn.org/stable/). diff --git a/api/Dockerfile b/api/Dockerfile new file mode 100644 index 0000000..eb464f7 --- /dev/null +++ b/api/Dockerfile @@ -0,0 +1,17 @@ +FROM node:19 + +# Install app dependencies +# A wildcard is used to ensure both package.json AND package-lock.json are copied +# where available (npm@5+) +COPY package*.json ./ + +RUN npm install +# If you are building your code for production +# RUN npm ci --only=production + +# Bundle app source +COPY . . + +EXPOSE 3001 + +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a21b0d3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3.8" +services: + oapen-engine : + build: ./oapen-engine/ + api: + build: ./api/ + expose: + - 3001 + ports: + - "127.0.0.1:3001:3001" + web: + build: ./web/ + expose: + - 3000 + ports: + - "127.0.0.1:3000:3000" +volumes: + db: + driver: local diff --git a/oapen-engine/Dockerfile b/oapen-engine/Dockerfile new file mode 100644 index 0000000..32ee0b5 --- /dev/null +++ b/oapen-engine/Dockerfile @@ -0,0 +1,36 @@ +FROM python:3.10-slim as base + +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 + + +FROM base AS python-deps + +# Install pipenv and compilation dependencies +RUN pip install pipenv +RUN apt-get update && apt-get install -y --no-install-recommends gcc linux-libc-dev + +# Install python dependencies in /.venv +COPY Pipfile . +RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy --skip-lock --verbose + + +FROM base AS runtime + +# Copy virtual env from python-deps stage +COPY --from=python-deps /.venv /.venv +ENV PATH="/.venv/bin:$PATH" + +# Create and switch to a new user +RUN useradd --create-home appuser +WORKDIR /home/appuser +USER appuser + +# Install application into container +COPY . . + +# Run the application +ENTRYPOINT ["python", "src/tasks/daemon.py"] \ No newline at end of file diff --git a/oapen-engine/Makefile b/oapen-engine/Makefile index 52c7991..663dd3d 100644 --- a/oapen-engine/Makefile +++ b/oapen-engine/Makefile @@ -34,3 +34,6 @@ run-tests: refresh-items: cd src && $(PYTHON) -m pipenv run python tasks/refresh_items.py + +run-daemon: + cd src && $(PYTHON) -m pipenv run python tasks/daemon.py \ No newline at end of file diff --git a/oapen-engine/Pipfile b/oapen-engine/Pipfile index e7d5b49..9e6633b 100644 --- a/oapen-engine/Pipfile +++ b/oapen-engine/Pipfile @@ -8,9 +8,8 @@ nltk = "*" requests = "*" psycopg2-binary = "*" pandas = "*" -sklearn = "*" +scikit-learn = "*" lxml = "*" -psutil = "*" [dev-packages] pytest = "*" diff --git a/oapen-engine/src/tasks/daemon.py b/oapen-engine/src/tasks/daemon.py new file mode 100644 index 0000000..b7cb6d0 --- /dev/null +++ b/oapen-engine/src/tasks/daemon.py @@ -0,0 +1,21 @@ +# Daemon to run processes in the background +import time +import sys, signal + + +def signal_handler(signal, frame): + print("\nprogram exiting gracefully") + sys.exit(0) + + +signal.signal(signal.SIGINT, signal_handler) + + +print("Daemon up and running") + +# TODO run cronjobs here +while True: + print("Daemon still running") + time.sleep(60) + +print("Daemon down") diff --git a/oapen-engine/src/tasks/seed.py b/oapen-engine/src/tasks/seed.py index ffcdf6a..d3eba02 100644 --- a/oapen-engine/src/tasks/seed.py +++ b/oapen-engine/src/tasks/seed.py @@ -10,7 +10,7 @@ import data.oapen as OapenAPI import model.ngrams as OapenEngine from data.connection import close_connection, get_connection from data.oapen_db import OapenDB -from util.kill_processes import kill_child_processes +# from util.kill_processes import kill_child_processes def ngrams_task(items): @@ -103,7 +103,7 @@ def main(): db_pool.shutdown(wait=False) io_pool.shutdown(wait=False) ngrams_pool.shutdown(wait=False) - kill_child_processes(os.getpid()) + # kill_child_processes(os.getpid()) close_connection(connection) for collection in collections: diff --git a/oapen-engine/src/util/kill_processes.py b/oapen-engine/src/util/kill_processes.py index 37b99ee..ecb2b24 100644 --- a/oapen-engine/src/util/kill_processes.py +++ b/oapen-engine/src/util/kill_processes.py @@ -1,13 +1,13 @@ import signal -import psutil +# import psutil -def kill_child_processes(parent_pid, sig=signal.SIGTERM): - try: - parent = psutil.Process(parent_pid) - except psutil.NoSuchProcess: - return - children = parent.children(recursive=True) - for process in children: - process.send_signal(sig) +# def kill_child_processes(parent_pid, sig=signal.SIGTERM): +# try: +# parent = psutil.Process(parent_pid) +# except psutil.NoSuchProcess: +# return +# children = parent.children(recursive=True) +# for process in children: +# process.send_signal(sig) diff --git a/web/Dockerfile b/web/Dockerfile new file mode 100644 index 0000000..310dc18 --- /dev/null +++ b/web/Dockerfile @@ -0,0 +1,19 @@ +FROM node:19 + +# Install app dependencies +# A wildcard is used to ensure both package.json AND package-lock.json are copied +# where available (npm@5+) +COPY package*.json ./ + +RUN npm install +# If you are building your code for production +# RUN npm ci --only=production + +# Bundle app source +COPY . . + +EXPOSE 3000 + +RUN npm run build + +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/web/next.config.js b/web/next.config.js new file mode 100644 index 0000000..20337c2 --- /dev/null +++ b/web/next.config.js @@ -0,0 +1,7 @@ +module.exports = { + typescript: { + // This ONLY is for type errors + // TODO remove when a better linting system is created + ignoreBuildErrors: true, + }, +}; diff --git a/web/pages/items/[uuid].tsx b/web/pages/items/[uuid].tsx index 5a555a1..72e5ef4 100644 --- a/web/pages/items/[uuid].tsx +++ b/web/pages/items/[uuid].tsx @@ -5,8 +5,8 @@ import { fetchSingleItemProps, SingleItemProps } from "../../lib/item/single"; export default function ItemSingle({ item }: SingleItemProps) { const name = - item.name || item.metadata.find(({ key }) => key == "grantor.name")?.value; - const type = item.metadata.find(({ key }) => key == "dc.type")?.value; + item?.name || item?.metadata.find(({ key }) => key == "grantor.name")?.value; + const type = item?.metadata.find(({ key }) => key == "dc.type")?.value; console.log({ item }); return ( <>