Skip to content

Commit 764b336

Browse files
ManishGuptaKumarManishGuptaKumar
committed
Changes for Datahub Api
1 parent 8795274 commit 764b336

File tree

148 files changed

+3487
-7528
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

148 files changed

+3487
-7528
lines changed

‎data_quality/src/main/resources/sql/006_dq_rules_asset_map.sql

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
CREATE SEQUENCE dq_rules_asset_map_seq
1+
CREATE SEQUENCE if not exists dq_rules_asset_map_seq
22
START WITH 1
33
INCREMENT BY 1
44
MINVALUE 1
@@ -8,7 +8,7 @@ CREATE SEQUENCE dq_rules_asset_map_seq
88

99

1010
CREATE TABLE dq_rules_asset_map (
11-
asset_map_id varchar(30) DEFAULT (('DQRM-'::text || lpad(nextval('sentinel.dq_rules_asset_map_seq'::regclass)::text, 20, '0'::text))) NOT NULL, -- Should be incremental
11+
asset_map_id varchar(30) DEFAULT (('DQRM-'::text || lpad(nextval('dq_rules_asset_map_seq'::regclass)::text, 20, '0'::text))) NOT NULL, -- Should be incremental
1212
rule_id varchar(30) NOT NULL, -- Rule Id
1313
database_name varchar(255) NOT NULL,
1414
schema_name varchar(255) NULL,
@@ -34,30 +34,29 @@ CREATE TABLE dq_rules_asset_map (
3434

3535
-- Column comments
3636

37-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.asset_map_id IS 'Should be incremental';
38-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.rule_id IS 'Rule Id';
39-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.database_name IS 'database_name';
40-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.effective_from IS 'Date from which this process will be effective';
41-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.expiry_date IS 'Date on Which this process was discontinued';
42-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.active_flg IS 'whether this record is still valid';
43-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.reserved_5 IS 'dummy column for future use';
44-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.reserved_4 IS 'dummy column for future use';
45-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.reserved_3 IS 'dummy column for future use';
46-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.reserved_2 IS 'dummy column for future use';
47-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.reserved_1 IS 'dummy column for future use';
48-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.updated_ts IS 'last updated by';
49-
COMMENT ON COLUMN sentinel.dq_rules_asset_map.updated_by IS 'last updated timestamp';
37+
COMMENT ON COLUMN dq_rules_asset_map.asset_map_id IS 'Should be incremental';
38+
COMMENT ON COLUMN dq_rules_asset_map.rule_id IS 'Rule Id';
39+
COMMENT ON COLUMN dq_rules_asset_map.database_name IS 'database_name';
40+
COMMENT ON COLUMN dq_rules_asset_map.effective_from IS 'Date from which this process will be effective';
41+
COMMENT ON COLUMN dq_rules_asset_map.expiry_date IS 'Date on Which this process was discontinued';
42+
COMMENT ON COLUMN dq_rules_asset_map.active_flg IS 'whether this record is still valid';
43+
COMMENT ON COLUMN dq_rules_asset_map.reserved_5 IS 'dummy column for future use';
44+
COMMENT ON COLUMN dq_rules_asset_map.reserved_4 IS 'dummy column for future use';
45+
COMMENT ON COLUMN dq_rules_asset_map.reserved_3 IS 'dummy column for future use';
46+
COMMENT ON COLUMN dq_rules_asset_map.reserved_2 IS 'dummy column for future use';
47+
COMMENT ON COLUMN dq_rules_asset_map.reserved_1 IS 'dummy column for future use';
48+
COMMENT ON COLUMN dq_rules_asset_map.updated_ts IS 'last updated by';
49+
COMMENT ON COLUMN dq_rules_asset_map.updated_by IS 'last updated timestamp';
5050

5151
-- Permissions
5252

5353
ALTER TABLE dq_rules_asset_map OWNER TO chimera_user;
5454
GRANT ALL ON TABLE dq_rules_asset_map TO chimera_user;
5555

5656

57-
-- sentinel.dq_rules_asset_map foreign keys
57+
-- dq_rules_asset_map foreign keys
5858

59-
ALTER TABLE sentinel.dq_rules_asset_map ADD CONSTRAINT dq_rules_asset_map_rule_id_fkey FOREIGN KEY (rule_id) REFERENCES data_quality_rules(rule_id);
59+
ALTER TABLE dq_rules_asset_map ADD CONSTRAINT dq_rules_asset_map_rule_id_fkey FOREIGN KEY (rule_id) REFERENCES data_quality_rules(rule_id);
6060

6161

6262

63-
drop table dq_rules_asset_map cascade

‎data_quality/src/main/resources/sql/008_data_quality_vw.sql

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
DROP view if exists sentinel.data_quality_vw;
2-
create or replace view sentinel.data_quality_vw as select
1+
DROP view if exists data_quality_vw;
2+
create or replace view data_quality_vw as select
33
ROW_NUMBER() OVER ( PARTITION BY table_name, database_name) AS rowNum,
44
dmp.process_name,
55
dc.control_name ,
@@ -12,8 +12,8 @@ asm.schema_name ,
1212
asm.table_name ,
1313
asm.partition_keys,
1414
asm.check_level as check_level
15-
from sentinel.dq_rules_asset_map asm join sentinel.data_quality_rules d on asm.rule_id=d.rule_id
16-
join sentinel.data_control_dimensions dcd on d.dimension_id =dcd.dimension_id
17-
join sentinel.data_controls dc on dcd.control_id =dc.control_id
18-
join sentinel.data_control_process_mapping cmp on cmp.control_id =dcd.control_id
19-
join sentinel.data_management_processes dmp on dmp.process_id =cmp.process_id ;
15+
from dq_rules_asset_map asm join data_quality_rules d on asm.rule_id=d.rule_id
16+
join data_control_dimensions dcd on d.dimension_id =dcd.dimension_id
17+
join data_controls dc on dcd.control_id =dc.control_id
18+
join data_control_process_mapping cmp on cmp.control_id =dcd.control_id
19+
join data_management_processes dmp on dmp.process_id =cmp.process_id ;

‎datahub/pom.xml

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,11 @@
5454
<artifactId>api-nexus-web</artifactId>
5555
<version>1.0-SNAPSHOT</version>
5656
</dependency>
57-
57+
<dependency>
58+
<groupId>io.kubernetes</groupId>
59+
<artifactId>client-java</artifactId>
60+
<version>18.0.1</version> <!-- Use the version matching your cluster -->
61+
</dependency>
5862
<dependency>
5963
<groupId>org.pantherslabs.chimera</groupId>
6064
<artifactId>api-nexus-security</artifactId>
@@ -164,6 +168,24 @@
164168
<groupId>org.apache.iceberg</groupId>
165169
<artifactId>iceberg-spark-runtime-3.5_2.12</artifactId>
166170
</dependency>
171+
<dependency>
172+
<groupId>org.testng</groupId>
173+
<artifactId>testng</artifactId>
174+
<version>7.11.0</version>
175+
<scope>test</scope>
176+
</dependency>
177+
<dependency>
178+
<groupId>org.testng</groupId>
179+
<artifactId>testng</artifactId>
180+
<version>7.11.0</version>
181+
<scope>test</scope>
182+
</dependency>
183+
<dependency>
184+
<groupId>org.pantherslabs.chimera</groupId>
185+
<artifactId>sentinel.data_quality</artifactId>
186+
<version>1.0-SNAPSHOT</version>
187+
<scope>compile</scope>
188+
</dependency>
167189

168190

169191
<!-- <dependency>
@@ -251,7 +273,7 @@
251273
</plugin>
252274
</plugins>
253275
<sourceDirectory>src/main/java</sourceDirectory>
254-
<testSourceDirectory>src/test/java</testSourceDirectory>
276+
<testSourceDirectory>src/test_disable/java</testSourceDirectory>
255277

256278
</build>
257279

‎datahub/src/main/chart/charts/nohup.out

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
Error from server (NotFound): pods "datahubcore-datahub-frontend-77548c857b-x55sl" not found
2+
Error from server (NotFound): pods "datahubcore-datahub-frontend-77548c857b-x55sl" not found
3+
Forwarding from 127.0.0.1:9002 -> 9002
4+
Forwarding from [::1]:9002 -> 9002
5+
Handling connection for 9002
6+
Handling connection for 9002
7+
Handling connection for 9002
8+
Handling connection for 9002
9+
Handling connection for 9002
10+
Handling connection for 9002
11+
Handling connection for 9002
12+
Handling connection for 9002
13+
Handling connection for 9002
14+
Handling connection for 9002
15+
Handling connection for 9002
16+
Handling connection for 9002
17+
Handling connection for 9002
18+
Handling connection for 9002
19+
Handling connection for 9002
20+
Handling connection for 9002
21+
Handling connection for 9002
22+
Handling connection for 9002
23+
Handling connection for 9002
24+
Handling connection for 9002
25+
Handling connection for 9002
26+
Handling connection for 9002
27+
Handling connection for 9002
28+
Handling connection for 9002
29+
Handling connection for 9002
30+
Handling connection for 9002
31+
Handling connection for 9002
32+
Handling connection for 9002
33+
Handling connection for 9002
34+
Handling connection for 9002
35+
E0729 12:46:59.146838 251655 portforward.go:413] "Unhandled Error" err="an error occurred forwarding 9002 -> 9002: error forwarding port 9002 to pod 36706da1c176319007670cc714dddfc32166e335171311c710a49ac958ad42d8, uid : Error response from daemon: No such container: 36706da1c176319007670cc714dddfc32166e335171311c710a49ac958ad42d8"
36+
E0729 12:46:59.148248 251655 portforward.go:413] "Unhandled Error" err="an error occurred forwarding 9002 -> 9002: error forwarding port 9002 to pod 36706da1c176319007670cc714dddfc32166e335171311c710a49ac958ad42d8, uid : Error response from daemon: No such container: 36706da1c176319007670cc714dddfc32166e335171311c710a49ac958ad42d8"
37+
error: lost connection to pod
38+
Error from server (NotFound): pods "datahubcore-datahub-frontend-55fd9b87f5-g6pjb" not found
39+
Forwarding from 127.0.0.1:9002 -> 9002
40+
Forwarding from [::1]:9002 -> 9002
41+
Handling connection for 9002
42+
Handling connection for 9002
43+
Handling connection for 9002
44+
Handling connection for 9002
45+
Handling connection for 9002
46+
Handling connection for 9002
47+
Handling connection for 9002
48+
Handling connection for 9002
49+
Handling connection for 9002
50+
Handling connection for 9002
51+
Handling connection for 9002
52+
Handling connection for 9002
53+
Handling connection for 9002
54+
Handling connection for 9002
55+
Handling connection for 9002
56+
Handling connection for 9002
57+
Handling connection for 9002
58+
Handling connection for 9002
59+
Handling connection for 9002
60+
Handling connection for 9002
61+
Handling connection for 9002
62+
Handling connection for 9002
63+
Handling connection for 9002
64+
Handling connection for 9002
65+
Handling connection for 9002
66+
Handling connection for 9002
67+
Handling connection for 9002
68+
Handling connection for 9002
69+
Handling connection for 9002
70+
E0729 14:06:00.395140 276068 portforward.go:413] "Unhandled Error" err="an error occurred forwarding 9002 -> 9002: error forwarding port 9002 to pod ad05e5e35d2c1b5e1925542dccb9e7de56b37b334544612ba1d2441ebee89c72, uid : Error response from daemon: No such container: ad05e5e35d2c1b5e1925542dccb9e7de56b37b334544612ba1d2441ebee89c72"
71+
E0729 14:06:00.395963 276068 portforward.go:413] "Unhandled Error" err="an error occurred forwarding 9002 -> 9002: error forwarding port 9002 to pod ad05e5e35d2c1b5e1925542dccb9e7de56b37b334544612ba1d2441ebee89c72, uid : Error response from daemon: No such container: ad05e5e35d2c1b5e1925542dccb9e7de56b37b334544612ba1d2441ebee89c72"
72+
error: lost connection to pod
73+
Unable to listen on port 8080: Listeners failed to create with the following errors: [unable to create listener: Error listen tcp4 127.0.0.1:8080: bind: address already in use unable to create listener: Error listen tcp6 [::1]:8080: bind: address already in use]
74+
error: unable to listen on any of the requested ports: [{8080 8080}]

‎datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/Constants.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ public class Constants {
8888
public static final String ML_PRIMARY_KEY_ENTITY_NAME = "mlPrimaryKey";
8989
public static final String POLICY_ENTITY_NAME = "dataHubPolicy";
9090
public static final String TAG_ENTITY_NAME = "tag";
91+
public static final String TAG_URN_PREFIX = "urn:li:tag:";
9192
public static final String CONTAINER_ENTITY_NAME = "container";
9293
public static final String DOMAIN_ENTITY_NAME = "domain";
9394
public static final String ER_MODEL_RELATIONSHIP_ENTITY_NAME = "erModelRelationship";
@@ -289,6 +290,8 @@ public class Constants {
289290
public static final String GLOSSARY_TERM_KEY_ASPECT_NAME = "glossaryTermKey";
290291
public static final String GLOSSARY_TERM_INFO_ASPECT_NAME = "glossaryTermInfo";
291292
public static final String GLOSSARY_RELATED_TERM_ASPECT_NAME = "glossaryRelatedTerms";
293+
public static final String GLOSSARY_TERM_URN_PREFIX = "urn:li:glossaryTerm:";
294+
292295

293296
// Glossary node
294297
public static final String GLOSSARY_NODE_KEY_ASPECT_NAME = "glossaryNodeKey";
@@ -323,6 +326,7 @@ public class Constants {
323326
public static final String ASSERTION_RUN_EVENT_ASPECT_NAME = "assertionRunEvent";
324327
public static final String ASSERTION_RUN_EVENT_STATUS_COMPLETE = "COMPLETE";
325328
public static final String ASSERTION_ACTIONS_ASPECT_NAME = "assertionActions";
329+
public static final String ASSERTION_URN_PREFIX = "urn:li:assertion:";
326330

327331
// Tests
328332
public static final String TEST_ENTITY_NAME = "test";
@@ -333,13 +337,17 @@ public class Constants {
333337
// Incident
334338
public static final String INCIDENT_KEY_ASPECT_NAME = "incidentKey";
335339
public static final String INCIDENT_INFO_ASPECT_NAME = "incidentInfo";
340+
public static final String INCIDENT_URN_PREFIX = "urn:li:incident:";
336341

337342
// DataHub Ingestion Source
338343
public static final String INGESTION_SOURCE_KEY_ASPECT_NAME = "dataHubIngestionSourceKey";
339344
public static final String INGESTION_INFO_ASPECT_NAME = "dataHubIngestionSourceInfo";
340345

341346
// DataHub Secret
342347
public static final String SECRET_VALUE_ASPECT_NAME = "dataHubSecretValue";
348+
public static final String DATAHUB_SECRET_URN_PREFIX = "urn:li:dataHubSecret:";
349+
public static final String DATAHUB_SECRET_ENTITY = "datahubsecret";
350+
343351

344352
// DataHub Execution Request
345353
public static final String EXECUTION_REQUEST_INPUT_ASPECT_NAME = "dataHubExecutionRequestInput";
@@ -379,6 +387,7 @@ public class Constants {
379387
// DataProduct
380388
public static final String DATA_PRODUCT_PROPERTIES_ASPECT_NAME = "dataProductProperties";
381389
public static final String DATA_PRODUCTS_ASPECT_NAME = "dataProducts";
390+
public static final String DATA_PRODUCTS_URN_PREFIX = "urn:li:dataProduct:";
382391

383392
// Ownership Types
384393
public static final String OWNERSHIP_TYPE_KEY_ASPECT_NAME = "ownershipTypeKey";

‎datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/DataHub.java renamed to ‎datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/DataHub.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
package org.pantherslabs.chimera.sentinel.datahub.api;
1+
package org.pantherslabs.chimera.sentinel.datahub;
22

33
import org.mybatis.spring.annotation.MapperScan;
44
import org.springframework.boot.SpringApplication;
55
import org.springframework.boot.autoconfigure.SpringBootApplication;
66
import org.springframework.context.annotation.ComponentScan;
77

88
@SpringBootApplication
9-
@ComponentScan(basePackages = "org.pantherslabs.chimera.sentinel.datahub.api")
9+
@ComponentScan(basePackages = "org.pantherslabs.chimera.sentinel.datahub")
1010
@MapperScan({
11-
"org.pantherslabs.chimera.sentinel.datahub.api.mapper.generated",
11+
"org.pantherslabs.chimera.sentinel.datahub.mapper.generated",
1212
"org.pantherslabs.chimera.unisca.api_nexus.api_nexus_client.dynamic_query.mapper"
1313
})
1414
public class DataHub {

‎datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/Field.java

Lines changed: 0 additions & 64 deletions
This file was deleted.

‎datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/GlossaryTerm.java

Lines changed: 0 additions & 39 deletions
This file was deleted.

0 commit comments

Comments
 (0)