Skip to content

Commit 4b1bf7c

Browse files
author
vikasrohit
authored
Merge pull request #431 from topcoder-platform/develop
Syncing master with v5 changes
2 parents 2d32e94 + bc60730 commit 4b1bf7c

File tree

347 files changed

+21690
-17619
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

347 files changed

+21690
-17619
lines changed

‎.circleci/config.yml

Lines changed: 44 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@ version: 2
22
python_env: &python_env
33
docker:
44
- image: circleci/python:2.7-stretch-browsers
5-
5+
66
install_awscli: &install_awscli
77
name: "Install awscli"
88
command: |
99
sudo pip install awscli --upgrade
1010
install_deploysuite: &install_deploysuite
11-
name: Installation of install_deploysuite.
12-
command: |
13-
git clone --branch master https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
14-
cp ./../buildscript/master_deploy.sh .
15-
cp ./../buildscript/buildenv.sh .
16-
cp ./../buildscript/awsconfiguration.sh .
17-
11+
name: Installation of install_deploysuite.
12+
command: |
13+
git clone --branch v1.4.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
14+
cp ./../buildscript/master_deploy.sh .
15+
cp ./../buildscript/buildenv.sh .
16+
cp ./../buildscript/awsconfiguration.sh .
17+
1818
# Instructions of deployment
1919
deploy_steps: &deploy_steps
2020
- checkout
@@ -23,22 +23,16 @@ deploy_steps: &deploy_steps
2323
- run: *install_awscli
2424
- run: *install_deploysuite
2525
- setup_remote_docker
26-
- run: docker build -t tc-project-service:latest .
26+
- run: docker build -t ${APPNAME}:latest .
2727
- deploy:
28-
name: "Running Masterscript - deploy tc-project-service "
28+
name: "Running Masterscript - deploy tc-project-service "
2929
command: |
30-
./awsconfiguration.sh $DEPLOY_ENV
30+
./awsconfiguration.sh $DEPLOY_ENV
3131
source awsenvconf
32-
./buildenv.sh -e $DEPLOY_ENV -b ${VAR_ENV}-tc-project-service-deployvar
32+
./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-deployvar
3333
source buildenvvar
34-
./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${VAR_ENV}-global-appvar,${VAR_ENV}-tc-project-service-appvar -i tc-project-service -p FARGATE
34+
./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
3535
36-
echo "======= Running Masterscript - deploy tc-project-service-consumers ==========="
37-
if [ -e ${VAR_ENV}-tc-project-service-appvar.json ]; then sudo rm -vf ${VAR_ENV}-tc-project-service-appvar.json; fi
38-
./buildenv.sh -e $DEPLOY_ENV -b ${VAR_ENV}-tc-project-service-consumers-deployvar
39-
source buildenvvar
40-
./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${VAR_ENV}-global-appvar,${VAR_ENV}-tc-project-service-appvar -i tc-project-service -p FARGATE
41-
4236
jobs:
4337
test:
4438
docker:
@@ -50,13 +44,20 @@ jobs:
5044
- image: elasticsearch:2.3
5145
- image: rabbitmq:3-management
5246
environment:
53-
DB_MASTER_URL: postgres://circle_test:@127.0.0.1:5432/circle_test
54-
AUTH_SECRET: secret
55-
AUTH_DOMAIN: topcoder-dev.com
56-
LOG_LEVEL: debug
57-
APP_VERSION: v4
47+
DEPLOY_ENV: "DEV"
48+
LOGICAL_ENV: "dev"
49+
APPNAME: "projects-api"
5850
steps:
5951
- checkout
52+
- run:
53+
name: "Install dependeency"
54+
command: |
55+
sudo apt update
56+
sudo apt install curl
57+
sudo apt install python-pip
58+
- run: *install_awscli
59+
- run: *install_deploysuite
60+
- setup_remote_docker
6061
- restore_cache:
6162
key: test-node-modules-{{ checksum "package.json" }}
6263
- run: npm install
@@ -65,39 +66,50 @@ jobs:
6566
paths:
6667
- node_modules
6768
- run: npm run lint
68-
- run: npm run test
69+
- run:
70+
name: "Running Masterscript - deploy tc-project-service "
71+
command: |
72+
./awsconfiguration.sh $DEPLOY_ENV
73+
source awsenvconf
74+
./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-testvar
75+
source buildenvvar
76+
npm run test
77+
rm -f buildenvvar
6978
- run: npm run build
7079
- persist_to_workspace:
7180
root: .
7281
paths:
7382
- dist
74-
83+
7584
deployProd:
7685
<<: *python_env
7786
environment:
7887
DEPLOY_ENV: "PROD"
79-
VAR_ENV: "prod"
88+
LOGICAL_ENV: "prod"
89+
APPNAME: "projects-api"
8090
steps: *deploy_steps
8191

8292
deployDev:
8393
<<: *python_env
8494
environment:
8595
DEPLOY_ENV: "DEV"
86-
VAR_ENV: "dev"
87-
steps: *deploy_steps
88-
96+
LOGICAL_ENV: "dev"
97+
APPNAME: "projects-api"
98+
steps: *deploy_steps
99+
89100
workflows:
90101
version: 2
91102
build:
92103
jobs:
93-
- test
104+
- test:
105+
context : org-global
94106
- deployDev:
95107
context : org-global
96108
requires:
97109
- test
98110
filters:
99111
branches:
100-
only: ['dev', 'dev-sts', 'feature/looker-api-integration']
112+
only: ['develop']
101113
- deployProd:
102114
context : org-global
103115
requires:

‎.nvmrc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
v8.2.1
1+
v8.9.4

‎README.md

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ Microservice to manage CRUD operations for all things Projects.
99
### Requirements
1010

1111
* [docker-compose](https://docs.docker.com/compose/install/) - We use docker-compose for running dependencies locally.
12-
* Nodejs 8.2.1 - consider using [nvm](https://github.com/creationix/nvm) or equivalent to manage your node version
12+
* Nodejs 8.9.4 - consider using [nvm](https://github.com/creationix/nvm) or equivalent to manage your node version
1313
* Install [libpg](https://www.npmjs.com/package/pg-native)
1414

1515
### Steps to run locally
@@ -54,6 +54,7 @@ Microservice to manage CRUD operations for all things Projects.
5454
```
5555
Alternatively, you may update `config/local.js` and replace `dockerhost` with your docker IP address.<br>
5656
You may try using command `docker-machine ip` to get your docker IP, but it works not for all systems.
57+
Also, be sure to update `busApiUrl` if you are running `tc-bus-api` locally. (See below)
5758

5859
Explanation of configs:
5960
- `config/mock.local.js` - Use local `mock-services` from docker to mock Identity and Member services instead of using deployed at Topcoder dev environment.
@@ -85,12 +86,66 @@ Microservice to manage CRUD operations for all things Projects.
8586
The project service will be served on `http://localhost:8001`.
8687

8788
### Import sample metadata & projects
89+
8890
```bash
8991
CONNECT_USER_TOKEN=<connect user token> npm run demo-data
9092
```
91-
This command will create sample metadata entries in the DB (duplicate what is currently in development environment).
93+
To retrieve data from DEV env we have to provide a valid user token (`CONNECT_USER_TOKEN`). You may login to http://connect.topcoder-dev.com and find the Bearer token in the request headers using browser dev tools.
94+
95+
This command for importing data uses API to create demo data. Which has a few pecularities:
96+
- data in DB would be for sure created
97+
- data in ElasticSearch Index (ES) would be only created if services [project-processor-es](https://github.com/topcoder-platform/project-processor-es) and [tc-bus-api](https://github.com/topcoder-platform/tc-bus-api) are also started locally. If you don't start them, then imported data wouldn't be indexed in ES, and would be only added to DB. You may start them locally separately, or better use `local/full/docker-compose.yml` as described [next section](#local-deployment-with-other-topcoder-services) which would start them automatically.
98+
- **NOTE** During data importing a lot of records has to be indexed in ES, so you have to wait about 5-10 minutes after `npm run demo-data` is finished until imported data is indexed in ES. You may watch logs of `project-processor-es` to see if its done or no.
99+
100+
### Local Deployment with other Topcoder Services.
101+
102+
* There exists an alternate `docker-compose.yml` file that can be used to spawn containers for the following services:
103+
104+
| Service | Name | Port |
105+
|----------|:-----:|:----:|
106+
| PostGreSQL DB | db | 5432 |
107+
| ElasticSearch | esearch | 9200,9300 |
108+
| RabbitMQ | queue | 5672, 15672 |
109+
| Zookeeper | zookeeper | 2181 |
110+
| Kafka | kafka | 9092 |
111+
| [tc-bus-api](https://github.com/topcoder-platform/tc-bus-api) | tc-bus-api | 8002 |
112+
| [project-processor-es](https://github.com/topcoder-platform/project-processor-es) | project-processor-es | 5000 |
113+
| [tc-notifications-api](https://github.com/topcoder-platform/tc-notifications) | tc-notifications-api | 4000 |
114+
| [tc-notifications-processor](https://github.com/topcoder-platform/tc-notifications) | tc-notifications-processor | 4001 |
115+
116+
* To have kafka create a list of desired topics on startup, there exists a file with the path `local/full/kafka-client/topics.txt`. Each line from the file will be added as a topic.
117+
* To run these services simply run the following commands:
118+
119+
```bash
120+
export AUTH0_CLIENT_ID=<insert required value here>
121+
export AUTH0_CLIENT_SECRET=<insert required value here>
122+
export AUTH0_URL=<insert required value here>
123+
export AUTH0_AUDIENCE=<insert required value here>
124+
export AUTH0_PROXY_SERVER_URL=<insert required value here>
125+
126+
cd local/full
127+
docker-compose up -d
128+
```
92129

93-
To retrieve data from DEV env we need to provide a valid user token. You may login to http://connect.topcoder-dev.com and find the Bearer token in the request headers using browser dev tools.
130+
* The environment variables specified in the commands above will be passed onto the containers that have been configured to read them.
131+
* The above command will start all containers in the background.
132+
* To view the logs of any of the services use the following command, replacing "SERVICE_NAME" with the corresponding value under the "Name" column in the above table:
133+
134+
```bash
135+
cd local/full
136+
docker-compose logs -f SERVICE_NAME
137+
```
138+
139+
* The containers have been configured such that all Topcoder services will wait until all the topics listed in `local/full/kafka-client/topics.txt` have been created. To monitor the progress of topic creation, you can view the logs of the `kafka-client` service, which will exit when all topics have been created.
140+
141+
* **WARNING**<br>
142+
After all the containers are started, make sure that `project-processor-es` service started successfully, as sometimes it doesn't start successfully as Kafka wasn't yet properly started at that moment. So run `docker-compose logs -f project-processor-es` to see its logs, you should see 3 lines with text `Subscribed to project.action.` like:
143+
```
144+
project-processor-es_1 | 2019-12-18T11:10:12.849Z DEBUG no-kafka-client Subscribed to project.action.update:0 offset 0 leader 96e65c46c746:9092
145+
project-processor-es_1 | 2019-12-18T11:10:12.851Z DEBUG no-kafka-client Subscribed to project.action.delete:0 offset 0 leader 96e65c46c746:9092
146+
project-processor-es_1 | 2019-12-18T11:10:12.852Z DEBUG no-kafka-client Subscribed to project.action.create:0 offset 0 leader 96e65c46c746:9092
147+
```
148+
If you don't see such lines, restart `project-processor-es` service ONLY by running `docker-compose restart project-processor-es`.
94149

95150
### Run Connect App with Project Service locally
96151

@@ -99,7 +154,9 @@ To be able to run [Connect App](https://github.com/appirio-tech/connect-app) wit
99154

100155
```js
101156
PROJECTS_API_URL: 'http://localhost:8001'
157+
TC_NOTIFICATION_URL: 'http://localhost:4000/v5/notifications' # if tc-notfication-api has been locally deployed
102158
```
159+
103160
2. Bypass token validation in Project Service.
104161

105162
In `tc-project-service/node_modules/tc-core-library-js/lib/auth/verifier.js` add this to line 23:
@@ -142,3 +199,33 @@ You can paste **swagger.yaml** to [swagger editor](http://editor.swagger.io/) o
142199

143200
#### Deploying without docker
144201
If you don't want to use docker to deploy to localhost. You can simply run `npm run start:dev` from root of project. This should start the server on default port `8001`.
202+
203+
### Kafka Commands
204+
205+
If you've used `docker-compose` with the file `local/full/docker-compose.yml` to spawn kafka & zookeeper, you can use the following commands to manipulate kafka topics and messages:
206+
(Replace TOPIC_NAME with the name of the desired topic)
207+
208+
**Create Topic**
209+
210+
```bash
211+
docker exec tc-projects-kafka /usr/bin/kafka-topics --create --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic TOPIC_NAME
212+
```
213+
214+
**List Topics**
215+
216+
```bash
217+
docker exec -it tc-projects-kafka /usr/bin/kafka-topics --list --zookeeper zookeeper:2181
218+
```
219+
220+
**Watch Topic**
221+
222+
```bash
223+
docker exec -it tc-projects-kafka /usr/bin/kafka-console-consumer --bootstrap-server localhost:9092 --zookeeper zookeeper:2181 --topic TOPIC_NAME
224+
```
225+
226+
**Post Message to Topic**
227+
228+
```bash
229+
docker exec -it tc-projects-kafka /usr/bin/kafka-console-producer --topic TOPIC_NAME --broker-list localhost:9092
230+
```
231+
The message can be passed using `stdin`

‎config/custom-environment-variables.json

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010
"host": "PROJECTS_ES_URL",
1111
"apiVersion": "2.3",
1212
"indexName": "PROJECTS_ES_INDEX_NAME",
13-
"docType": "projectV4",
13+
"docType": "PROJECTS_ES_DOC_TYPE",
1414
"timelineIndexName": "TIMELINES_ES_INDEX_NAME",
15-
"timelineDocType": "TIMELINES_ES_DOC_TYPE"
15+
"timelineDocType": "TIMELINES_ES_DOC_TYPE",
16+
"metadataIndexName": "METADATA_ES_INDEX_NAME",
17+
"metadataDocType": "METADATA_ES_DOC_TYPE"
1618
},
1719
"rabbitmqURL": "RABBITMQ_URL",
1820
"pubsubQueueName": "PUBSUB_QUEUE_NAME",
@@ -50,6 +52,7 @@
5052
"accountsAppUrl": "ACCOUNTS_APP_URL",
5153
"inviteEmailSubject": "INVITE_EMAIL_SUBJECT",
5254
"inviteEmailSectionTitle": "INVITE_EMAIL_SECTION_TITLE",
55+
"pageSize": "PAGE_SIZE",
5356
"SSO_REFCODES": "SSO_REFCODES",
5457
"lookerConfig": {
5558
"BASE_URL": "LOOKER_API_BASE_URL",

‎config/default.json

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"apiVersion": "v4",
2+
"apiVersion": "v5",
33
"AUTH_SECRET": "secret",
44
"logLevel": "info",
5-
"version": "v4",
5+
"version": "v5",
66
"captureLogs": "false",
77
"enableFileUpload": "true",
88
"logentriesToken": "",
@@ -19,11 +19,13 @@
1919
"projectAttachmentPathSuffix": "attachments",
2020
"elasticsearchConfig": {
2121
"host": "",
22-
"apiVersion": "2.3",
22+
"apiVersion": "6.8",
2323
"indexName": "projects",
24-
"docType": "projectV4",
24+
"docType": "doc",
2525
"timelineIndexName": "timelines",
26-
"timelineDocType": "timelineV4"
26+
"timelineDocType": "doc",
27+
"metadataIndexName": "metadata",
28+
"metadataDocType": "doc"
2729
},
2830
"connectProjectUrl":"",
2931
"dbConfig": {
@@ -44,13 +46,8 @@
4446
"busApiToken": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicHJvamVjdC1zZXJ2aWNlIiwiaWF0IjoxNTEyNzQ3MDgyLCJleHAiOjE1MjEzODcwODJ9.PHuNcFDaotGAL8RhQXQMdpL8yOKXxjB5DbBIodmt7RE",
4547
"HEALTH_CHECK_URL": "_health",
4648
"maxPhaseProductCount": 1,
47-
"AUTH0_CLIENT_ID": "",
48-
"AUTH0_CLIENT_SECRET": "",
49-
"AUTH0_AUDIENCE": "",
50-
"AUTH0_URL": "",
51-
"TOKEN_CACHE_TIME": "",
49+
"TOKEN_CACHE_TIME": "86000",
5250
"whitelistedOriginsForUserIdAuth": "[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\"]",
53-
"AUTH0_PROXY_SERVER_URL" : "",
5451
"EMAIL_INVITE_FROM_NAME":"Topcoder",
5552
"EMAIL_INVITE_FROM_EMAIL":"[email protected]",
5653
"inviteEmailSubject": "You are invited to Topcoder",
@@ -59,8 +56,9 @@
5956
"accountsAppUrl": "https://accounts.topcoder-dev.com",
6057
"MAX_REVISION_NUMBER": 100,
6158
"UNIQUE_GMAIL_VALIDATION": false,
62-
"SSO_REFCODES": "[]",
59+
"pageSize": 20,
6360
"VALID_STATUSES_BEFORE_PAUSED": "[\"active\"]",
61+
"SSO_REFCODES": "[]",
6462
"lookerConfig": {
6563
"BASE_URL": "",
6664
"CLIENT_ID": "",

‎config/m2m.local.js

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ if (process.env.NODE_ENV === 'test') {
55
config = require('./test.json');
66
} else {
77
config = {
8+
busApiUrl: "http://localhost:8002/v5",
89
identityServiceEndpoint: "https://api.topcoder-dev.com/v3/",
910
authSecret: 'secret',
1011
authDomain: 'topcoder-dev.com',
@@ -23,11 +24,7 @@ if (process.env.NODE_ENV === 'test') {
2324
idleTimeout: 1000,
2425
},
2526
elasticsearchConfig: {
26-
host: 'dockerhost:9200',
27-
// target elasticsearch 2.3 version
28-
apiVersion: '2.3',
29-
indexName: 'projects',
30-
docType: 'projectV4'
27+
host: 'dockerhost:9200'
3128
},
3229
whitelistedOriginsForUserIdAuth: "[\"\"]",
3330
};

‎config/mock.local.js

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ if (process.env.NODE_ENV === 'test') {
55
config = require('./test.json');
66
} else {
77
config = {
8+
busApiUrl: "http://localhost:8002/v5",
89
identityServiceEndpoint: "http://dockerhost:3001/",
910
authSecret: 'secret',
1011
authDomain: 'topcoder-dev.com',
@@ -23,11 +24,7 @@ if (process.env.NODE_ENV === 'test') {
2324
idleTimeout: 1000,
2425
},
2526
elasticsearchConfig: {
26-
host: 'dockerhost:9200',
27-
// target elasticsearch 2.3 version
28-
apiVersion: '2.3',
29-
indexName: 'projects',
30-
docType: 'projectV4'
27+
host: 'dockerhost:9200'
3128
},
3229
whitelistedOriginsForUserIdAuth: "[\"\"]",
3330
};

0 commit comments

Comments
 (0)