Merge pull request #64 from arXiv/develop

Pre-release merge for submission core v0.7.1

Author: erickpeirson

.coveragerc

@@ -4,5 +4,6 @@ omit =
     setup.py
     docs/*
     test*
+    */test*
     */app.py
     */wsgi.py

.gitignore

@@ -100,6 +100,8 @@ ENV/
 # mypy
 .mypy_cache/
 
+.pytest_cache/
+
 .DS_Store
 
 submissions.tsv

.travis.yml

@@ -2,7 +2,7 @@ language: python
 sudo: required
 services:
   - docker
-cache: pip
+#cache: pip
 env:
   MIN_PYLINT_SCORE: 8
 os:
@@ -11,10 +11,11 @@ python:
   - "3.6"
 script:
   - pip install pipenv
-  - pipenv install
   - pipenv install --dev
-  - pipenv run nose2 --with-coverage
-  - tests/lint.sh
-  - tests/docstyle.sh
+  - pipenv install ./core
+  - pipenv run pip show arxiv-base
+  - pipenv run pytest --cov=agent/agent --cov=core/arxiv --cov-report=term-missing agent/agent core/arxiv
 after_success:
   - coveralls
+#  - tests/lint.sh
+  - tests/docstyle.sh

Dockerfile

@@ -0,0 +1,24 @@
+# arxiv/submission-agent
+
+ARG BASE_VERSION=ARXIVNG-2462
+
+FROM arxiv/base:${BASE_VERSION}
+
+WORKDIR /opt/arxiv
+
+ENV KINESIS_STREAM="SubmissionEvents" \
+    KINESIS_SHARD_ID="0" \
+    KINESIS_START_TYPE="TRIM_HORIZON" \
+    SUBMISSION_AGENT_DATABASE_URI="" \
+    LOGLEVEL=10 \
+    JWT_SECRET="foo"
+
+COPY Pipfile Pipfile.lock /opt/arxiv/
+COPY core/ /opt/arxiv/core/
+RUN pipenv install /opt/arxiv/core/ && rm -rf ~/.cache/pip
+COPY agent/agent/ /opt/arxiv/agent/
+
+ENTRYPOINT ["pipenv", "run"]
+
+CMD ["python", "-m", "agent.consumer"]
+# CMD ["celery", "worker", "-A", "agent.worker.worker_app", "--loglevel=INFO", "-E", "--concurrency=2"]

Dockerfile-metadata

@@ -1,6 +1,6 @@
 # arxiv/submission-api
 
-FROM arxiv/base:0.7.1
+FROM arxiv/base
 
 WORKDIR /opt/arxiv
 
@@ -12,14 +12,14 @@ ENV LANG en_US.utf8
 RUN yum -y install mariadb-devel
 
 # Add Python consumer and configuration.
-ADD Pipfile /opt/arxiv/Pipfile
-ADD Pipfile.lock /opt/arxiv/Pipfile.lock
+ADD metadata/Pipfile /opt/arxiv/Pipfile
 RUN pip install pipenv
 RUN pipenv install
 
+ADD core/ /opt/arxiv/core/
+RUN pipenv install /opt/arxiv/core/
+
 ADD metadata/ /opt/arxiv/
-ADD authorization/authorization/ /opt/arxiv/authorization/
-ADD core/events/ /opt/arxiv/events/
 ADD metadata/bin/entrypoint.sh /opt/arxiv/entrypoint.sh
 
 ENV JWT_SECRET "foo"
@@ -29,5 +29,6 @@ EXPOSE 8000
 
 CMD ./entrypoint.sh --http-socket :8000 -w wsgi -M \
         -t 3000 --manage-script-name \
+        --buffer-size 65535 \
         --processes 8 --threads 1 --async 100 --ugreen \
         --mount /submission=wsgi.py

Pipfile

@@ -1,32 +1,50 @@
 [[source]]
-
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-
 [packages]
-
-flask = "*"
-dataclasses = "*"
-pyjwt = "*"
-jsonschema = "*"
-arxiv-base = "==0.7.1"
-sqlalchemy = "*"
-pyyaml = "*"
-pytz = "*"
-uwsgi = "*"
-mysqlclient = "*"
-mimesis = "*"
-
+flask = "==1.0.2"
+dataclasses = "==0.6"
+pyjwt = "==1.6.4"
+jsonschema = "==2.6.0"
+sqlalchemy = ">=1.3.0"
+pytz = "==2018.7"
+uwsgi = "==2.0.17.1"
+mysqlclient = "==1.3.13"
+mimesis = "==2.1.0"
+bleach = ">=3.0.2"
+python-dateutil = "*"
+unidecode = "*"
+celery = "==4.1.0"
+arxiv-auth = "==0.3.2rc5"
+mypy_extensions = "*"
+kombu = "==4.1.0"
+semver = "*"
+flask-sqlalchemy = "*"
+retry = "*"
+backports-datetime-fromisoformat = "*"
+arxiv-base = "==0.15.8rc4"
+alembic = "*"
+urllib3 = ">=1.24.2"
+Jinja2 = ">=2.10.1"
+pyyaml = ">=4.2b1"
+arxiv-vault = "==0.1.1rc15"
+redis = "==2.10.6"
 
 [dev-packages]
-
 "nose2" = "*"
 openapi-spec-validator = "*"
-coverage = "*"
+coverage = "==4.5"
 coveralls = "*"
-pylint = "*"
-pydocstyle = "*"
-mypy = "*"
+pylint = "<2"
+mypy = "==0.660"
 mimesis = "*"
+sphinx = "*"
+sphinx-autodoc-typehints = "*"
+pydocstyle = "==3.0.0"
+pytest = "*"
+pytest-cov = "*"
+
+[pipenv]
+allow_prereleases = true