DataHub Docker & Helm Chart

Author: ManishGuptaKumar

datahub/pom.xml

@@ -60,7 +60,11 @@
             <artifactId>api-nexus-security</artifactId>
             <version>1.0-SNAPSHOT</version>
         </dependency>
-
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.pantherslabs.chimera</groupId>
             <artifactId>api-nexus-db</artifactId>

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/Constants.java

@@ -60,6 +60,8 @@ public class Constants {
         // been processed and
         // should not be reprocessed
         public static final String APP_SOURCE = "appSource";
+        public static final String APP_URN_PREFIX = "urn:li:application:";
+        public static final String APPLICATION_ENTITY_NAME = "application";
 
         // App sources
         public static final String UI_SOURCE = "ui";

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/DataHub.java

@@ -7,8 +7,10 @@
 
 @SpringBootApplication
 @ComponentScan(basePackages = "org.pantherslabs.chimera.sentinel.datahub.api")
-@MapperScan("org.pantherslabs.chimera.sentinel.datahub.api.mapper.generated")
-@MapperScan("org.pantherslabs.chimera.unisca.api_nexus.api_nexus_client.dynamic_query.mapper")
+@MapperScan({
+        "org.pantherslabs.chimera.sentinel.datahub.api.mapper.generated",
+        "org.pantherslabs.chimera.unisca.api_nexus.api_nexus_client.dynamic_query.mapper"
+})
 public class DataHub {
     public static void main(String[] args) {
         SpringApplication.run(DataHub.class, args);

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/controller/DatasetController.java

@@ -0,0 +1,49 @@
+package org.pantherslabs.chimera.sentinel.datahub.api.controller;
+
+import org.pantherslabs.chimera.sentinel.datahub.api.service.ManageDatasets;
+import org.pantherslabs.chimera.sentinel.datahub.modal.EmitResult;
+import org.pantherslabs.chimera.unisca.api_nexus.api_nexus_client.response.SuccessResponse;
+import org.pantherslabs.chimera.unisca.exception.ChimeraException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.http.ResponseEntity;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@RestController
+@RequestMapping("/api/dataset")
+public class DatasetController {
+
+    @Autowired
+    ManageDatasets manageDatasets;
+    /**
+     * Accepts a dataset definition as JSON string (raw body) and a createdBy query parameter.
+     */
+
+    @PostMapping("/create")
+    public ResponseEntity<?> createDataset(
+            @RequestBody String datasetDefinitionJson,
+            @RequestParam(name = "inFormat", required = false, defaultValue = "json") String inFormat) {
+        EmitResult result = null;
+        try {
+            result = manageDatasets.createDataset(datasetDefinitionJson, inFormat, "UPSERT");
+            List<SuccessResponse> summaries = result.getSucceeded().stream()
+                    .map(p -> new SuccessResponse(
+                            p.getEntityUrn().toString() + " " + p.getAspectName() + " created successfully..", "200"))
+                    .collect(Collectors.toList());
+            return ResponseEntity.ok(summaries);
+        } catch (Exception e) {
+            assert result != null;
+            throw new ChimeraException(
+                    "APIException." + result.getErrorDetails().getStatus(),
+                    Map.of("exception", result.getErrorDetails().getMessage()),
+                    null,
+                    HttpStatus.valueOf(result.getErrorDetails().getStatus())
+            );
+        }
+    }
+
+}

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/controller/OwnershipController.java

@@ -42,7 +42,7 @@ public ResponseEntity<?> addOwners(@RequestBody OwnerRequest request) {
         EmitResult result = null;
         try {
             Urn urn = Urn.createFromString(request.getEntityUrn());
-            result = manageOwnership.createOwners(
+            result = ManageOwnership.createOwners(
                     urn,
                     request.getEntityType(),
                     request.getOwnersInfo(),

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/Dataset.java

@@ -0,0 +1,49 @@
+package org.pantherslabs.chimera.sentinel.datahub.api.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Getter
+@Setter
+public class Dataset {
+
+    // Required fields
+    @NotNull
+    private String dataProductName;
+
+    @NotNull
+    private String name;
+
+    @NotNull
+    private String fabricType;
+
+    @NotNull
+    private String datasetPlatformName;
+
+    // Optional fields
+    private String displayName;
+    private String description;
+    private String qualifiedName;
+    private String uri;
+    private String domain;
+
+    private List<Tag> tags;
+    private List<Property> properties;
+    private List<Owners> owners;
+    private List<GlossaryTerm> glossaryTerm;
+    private List<Field> fields;
+
+    // Constructors
+    public Dataset() {}
+
+    public Dataset(@NotNull String dataProductName, @NotNull String name, @NotNull String fabricType,
+                   @NotNull String datasetPlatformName) {
+        this.dataProductName = dataProductName;
+        this.name = name;
+        this.fabricType = fabricType;
+        this.datasetPlatformName = datasetPlatformName;
+    }
+}

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/Field.java

@@ -0,0 +1,64 @@
+package org.pantherslabs.chimera.sentinel.datahub.api.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+import lombok.Setter;
+import org.pantherslabs.chimera.sentinel.datahub.modal.ForeignKey;
+import org.pantherslabs.chimera.sentinel.datahub.modal.GlossaryTerm;
+import org.pantherslabs.chimera.sentinel.datahub.modal.Tag;
+
+import java.util.List;
+
+@Getter
+@Setter
+//@JsonIgnoreProperties(ignoreUnknown = true)
+
+public class Field {
+
+    public Field(){}
+
+    public Field(String name, String type, String displayName, String description, String fieldCanonicalName,
+                 int maxLength, boolean isPartitionKey, boolean isPrimaryKey, boolean isSampleTime, boolean isNullable,
+                 List<ForeignKey> foreignKey, List<org.pantherslabs.chimera.sentinel.datahub.modal.Tag> tags, List<org.pantherslabs.chimera.sentinel.datahub.modal.GlossaryTerm> glossaryTerm) {
+        this.name = name;
+        this.type = type;
+        this.displayName = displayName;
+        this.description = description;
+        this.fieldCanonicalName = fieldCanonicalName;
+        this.maxLength = maxLength;
+        this.isPartitionKey = isPartitionKey;
+        this.isPrimaryKey = isPrimaryKey;
+        this.isSampleTime = isSampleTime;
+        this.isNullable = isNullable;
+        this.foreignKey = foreignKey;
+        this.tags = tags;
+        this.glossaryTerm = glossaryTerm;
+    }
+
+    @JsonProperty("name")
+    private String name;
+    @JsonProperty("type")
+    private String type;
+    @JsonProperty("displayName")
+    private String displayName;
+    @JsonProperty("description")
+    private String description;
+    @JsonProperty("fieldCanonicalName")
+    private String fieldCanonicalName;
+    @JsonProperty("maxLength")
+    private int maxLength;
+    @JsonProperty("isPartitionKey")
+    private boolean isPartitionKey;
+    @JsonProperty("isPrimaryKey")
+    private boolean isPrimaryKey;
+    @JsonProperty("isSampleTime")
+    private boolean isSampleTime;
+    @JsonProperty("isNullable")
+    private boolean isNullable;
+    @JsonProperty("foreignKey")
+    private List<ForeignKey> foreignKey;
+    @JsonProperty("tags")
+    private List<Tag> tags;
+    @JsonProperty("glossaryTerm")
+    private List<GlossaryTerm> glossaryTerm;
+}

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/GlossaryTerm.java

@@ -0,0 +1,39 @@
+package org.pantherslabs.chimera.sentinel.datahub.api.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.pantherslabs.chimera.sentinel.datahub.modal.GlossaryNodeGroup;
+import org.pantherslabs.chimera.sentinel.datahub.modal.GlossaryRelatedTerm;
+
+import java.util.List;
+import java.util.Map;
+
+@Getter
+@Setter
+public class GlossaryTerm {
+
+    public GlossaryTerm(){}
+
+    public GlossaryTerm(String glossaryTermName, String documentations, String parentTermName, String sourceRef,
+                        String sourceURL, GlossaryNodeGroup glossaryNodeRecord, List<GlossaryRelatedTerm>
+                                glossaryRelatedTermsRecord, Map<String, String> customProperties) {
+        this.glossaryTermName = glossaryTermName;
+        this.documentations = documentations;
+        this.parentTermName = parentTermName;
+        this.sourceRef = sourceRef;
+        this.sourceURL = sourceURL;
+        this.glossaryNodeRecord = glossaryNodeRecord;
+        this.glossaryRelatedTermsRecord = glossaryRelatedTermsRecord;
+        this.customProperties = customProperties;
+    }
+
+    private String glossaryTermName;
+    private String documentations;
+    private String parentTermName;
+    private String sourceRef;
+    private String sourceURL;
+    private GlossaryNodeGroup glossaryNodeRecord;
+    private List<GlossaryRelatedTerm> glossaryRelatedTermsRecord;
+    private Map<String, String> customProperties;
+
+}

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/Owners.java

@@ -0,0 +1,50 @@
+package org.pantherslabs.chimera.sentinel.datahub.api.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.validation.constraints.NotNull;
+import java.util.Locale;
+import java.util.Set;
+
+@Getter
+@Setter
+public class Owners {
+
+    // Valid owners set as a constant
+    private static final Set<String> VALID_OWNERS = Set.of(
+            "CUSTOM", "TECHNICAL_OWNER", "BUSINESS_OWNER", "DATA_STEWARD",
+            "NONE", "DEVELOPER", "DATAOWNER", "PRODUCER",
+            "DELEGATE", "CONSUMER", "STAKEHOLDER", "$UNKNOWN"
+    );
+
+    // Get the name of the owner
+    @Getter
+    @NotNull
+    private String name;
+
+    public Owners(){}
+
+    public Owners(String name, String type) {
+        this.name = name;
+        this.type = type;
+    }
+
+    @NotNull
+    private String type;
+
+    // Get the type of the owner, with validation logic
+    public String getType() {
+        // Check if type is null or empty before proceeding
+        if (type == null || type.trim().isEmpty()) {
+            return "urn:li:ownershipType:__system__none";
+        }
+
+        return VALID_OWNERS.contains(type.toUpperCase(Locale.ROOT)) ?
+                "urn:li:ownershipType:__system__" + type.toLowerCase(Locale.ROOT) :
+                "urn:li:ownershipType:__system__none";
+
+    }
+}
+
+

datahub/src/main/java/org/pantherslabs/chimera/sentinel/datahub/api/dto/Property.java

@@ -0,0 +1,26 @@
+package org.pantherslabs.chimera.sentinel.datahub.api.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.validation.constraints.NotNull;
+
+@Setter
+@Getter
+public class Property {
+
+    // Getters and Setters
+    @NotNull
+    private String name;
+
+    @NotNull
+    private String value;
+
+    public Property(){}
+    // Constructor
+    public Property(String name, String value) {
+        this.name = name;
+        this.value = value;
+    }
+
+}