Pre-release merge for submission core v0.7.1

.coveragerc

@@ -4,5 +4,6 @@ omit =
     setup.py
     docs/*
     test*
+    */test*
     */app.py
     */wsgi.py

.travis.yml

@@ -2,7 +2,7 @@ language: python
 sudo: required
 services:
   - docker
-cache: pip
+#cache: pip
 env:
   MIN_PYLINT_SCORE: 8
 os:
@@ -11,10 +11,11 @@ python:
   - "3.6"
 script:
   - pip install pipenv
-  - pipenv install
   - pipenv install --dev
-  - pipenv run nose2 --with-coverage
-  - tests/lint.sh
-  - tests/docstyle.sh
+  - pipenv install ./core
+  - pipenv run pip show arxiv-base
+  - pipenv run pytest --cov=agent/agent --cov=core/arxiv --cov-report=term-missing agent/agent core/arxiv
 after_success:
   - coveralls
+#  - tests/lint.sh
+  - tests/docstyle.sh

Dockerfile-agent

@@ -0,0 +1,33 @@
+# arxiv/submission-agent
+
+FROM arxiv/base
+
+WORKDIR /opt/arxiv
+
+ENV LC_ALL en_US.utf8
+ENV LANG en_US.utf8
+
+# We're using MariaDB for testing/development, but production is MySQL (for
+# now).
+RUN yum -y install mariadb-devel
+
+# Add Python consumer and configuration.
+RUN pip install pipenv
+RUN pipenv install
+
+ENV AWS_ACCESS_KEY_ID ""
+ENV AWS_SECRET_ACCESS_KEY ""
+
+ADD core/ /opt/arxiv/core/
+RUN pipenv install /opt/arxiv/core/
+ADD agent/agent/ /opt/arxiv/agent/
+
+ENV KINESIS_STREAM "SubmissionEvents"
+ENV KINESIS_SHARD_ID "0"
+ENV KINESIS_START_TYPE "TRIM_HORIZON"
+ENV SUBMISSION_AGENT_DATABASE_URI ""
+ENV LOGLEVEL 10
+
+ENV JWT_SECRET "foo"
+
+ENTRYPOINT ["pipenv", "run", "python3.6", "-m", "agent.consumer"]

Dockerfile-metadata

@@ -1,6 +1,6 @@
 # arxiv/submission-api
 
-FROM arxiv/base:0.7.1
+FROM arxiv/base
 
 WORKDIR /opt/arxiv
 
@@ -12,14 +12,14 @@ ENV LANG en_US.utf8
 RUN yum -y install mariadb-devel
 
 # Add Python consumer and configuration.
-ADD Pipfile /opt/arxiv/Pipfile
-ADD Pipfile.lock /opt/arxiv/Pipfile.lock
+ADD metadata/Pipfile /opt/arxiv/Pipfile
 RUN pip install pipenv
 RUN pipenv install
 
+ADD core/ /opt/arxiv/core/
+RUN pipenv install /opt/arxiv/core/
+
 ADD metadata/ /opt/arxiv/
-ADD authorization/authorization/ /opt/arxiv/authorization/
-ADD core/events/ /opt/arxiv/events/
 ADD metadata/bin/entrypoint.sh /opt/arxiv/entrypoint.sh
 
 ENV JWT_SECRET "foo"
@@ -29,5 +29,6 @@ EXPOSE 8000
 
 CMD ./entrypoint.sh --http-socket :8000 -w wsgi -M \
         -t 3000 --manage-script-name \
+        --buffer-size 65535 \
         --processes 8 --threads 1 --async 100 --ugreen \
         --mount /submission=wsgi.py

Dockerfile-worker

@@ -0,0 +1,26 @@
+# arxiv/submission-worker
+
+FROM arxiv/base
+
+WORKDIR /opt/arxiv
+
+ENV LC_ALL en_US.utf8
+ENV LANG en_US.utf8
+
+# We're using MariaDB for testing/development, but production is MySQL (for
+# now).
+RUN yum -y install mariadb-devel
+
+# Add Python consumer and configuration.
+RUN pip install pipenv
+RUN pipenv install
+
+ADD core/ /opt/arxiv/core/
+RUN pipenv install /opt/arxiv/core/
+ADD agent/agent/ /opt/arxiv/agent/
+
+ENV JWT_SECRET "foo"
+EXPOSE 8000
+
+ENTRYPOINT ["pipenv", "run", "celery", "worker"]
+CMD ["-A", "agent.worker.worker_app", "--loglevel=INFO", "-E", "--concurrency=2"]

Pipfile

@@ -1,32 +1,50 @@
 [[source]]
-
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-
 [packages]
-
-flask = "*"
-dataclasses = "*"
-pyjwt = "*"
-jsonschema = "*"
-arxiv-base = "==0.7.1"
-sqlalchemy = "*"
-pyyaml = "*"
-pytz = "*"
-uwsgi = "*"
-mysqlclient = "*"
-mimesis = "*"
-
+flask = "==1.0.2"
+dataclasses = "==0.6"
+pyjwt = "==1.6.4"
+jsonschema = "==2.6.0"
+sqlalchemy = ">=1.3.0"
+pytz = "==2018.7"
+uwsgi = "==2.0.17.1"
+mysqlclient = "==1.3.13"
+mimesis = "==2.1.0"
+bleach = ">=3.0.2"
+python-dateutil = "*"
+unidecode = "*"
+celery = "==4.1.0"
+arxiv-auth = "==0.3.2rc5"
+mypy_extensions = "*"
+kombu = "==4.1.0"
+semver = "*"
+flask-sqlalchemy = "*"
+retry = "*"
+backports-datetime-fromisoformat = "*"
+arxiv-base = "==0.15.7rc8"
+arxiv-submission-core = {path = "./core"}
+alembic = "*"
+urllib3 = ">=1.24.2"
+Jinja2 = ">=2.10.1"
+pyyaml = ">=4.2b1"
 
 [dev-packages]
-
 "nose2" = "*"
 openapi-spec-validator = "*"
 coverage = "*"
 coveralls = "*"
-pylint = "*"
-pydocstyle = "*"
-mypy = "*"
+pylint = "<2"
+mypy = "==0.660"
 mimesis = "*"
+sphinx = "*"
+sphinx-autodoc-typehints = "*"
+pydocstyle = "==3.0.0"
+pytest = "*"
+pytest-cov = "*"
+sphinx-jekyll-builder = "*"
+
+[pipenv]
+allow_prereleases = true