-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
1292df3
commit 6a23e48
Showing
14 changed files
with
362 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| # ignore already-built apps | ||
| output | ||
| output/* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # ignore .last_id file, used to store the last id of the decision stored when calling curl | ||
| .last_id | ||
|
|
||
| # output folder, used to build the app outside of docker if needed | ||
| /output | ||
|
|
||
| .DS_Store |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package api | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/ioutil" | ||
| "marble/marble-backend/app" | ||
| "net/http" | ||
|
|
||
| "github.com/go-chi/chi/v5" | ||
| ) | ||
|
|
||
| func (a *API) handleIngestion() http.HandlerFunc { | ||
|
|
||
| /////////////////////////////// | ||
| // Request and Response types defined in scope | ||
| /////////////////////////////// | ||
|
|
||
| // return is a decision | ||
|
|
||
| return func(w http.ResponseWriter, r *http.Request) { | ||
|
|
||
| /////////////////////////////// | ||
| // Authorize request | ||
| /////////////////////////////// | ||
| orgID, err := orgIDFromCtx(r.Context()) | ||
| if err != nil { | ||
| http.Error(w, "", http.StatusUnauthorized) | ||
| return | ||
| } | ||
|
|
||
| object_type := chi.URLParam(r, "object_type") | ||
| fmt.Printf("Received object type: %s\n", object_type) | ||
|
|
||
| body, err := ioutil.ReadAll(r.Body) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| object_body := body | ||
| fmt.Printf("Received object body: %s\n", object_body) | ||
|
|
||
| a.app.IngestObject(orgID, app.IngestPayload{ObjectType: object_type, ObjectBody: object_body}) | ||
|
|
||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package app | ||
|
|
||
| import ( | ||
| "fmt" | ||
| ) | ||
|
|
||
| type IngestPayload struct { | ||
| ObjectType string | ||
| ObjectBody []byte | ||
| } | ||
|
|
||
| func (a *App) IngestObject(organizationID string, ingestPayload IngestPayload) (err error) { | ||
| fmt.Println(ingestPayload) | ||
| return a.repository.IngestObject(organizationID, ingestPayload) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| package app | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "log" | ||
| "strings" | ||
| "time" | ||
| "unicode" | ||
|
|
||
| "github.com/go-playground/validator" | ||
| dynamicstruct "github.com/ompluscator/dynamic-struct" | ||
| ) | ||
|
|
||
| func capitalize(str string) string { | ||
| runes := []rune(str) | ||
| runes[0] = unicode.ToUpper(runes[0]) | ||
| return string(runes) | ||
| } | ||
|
|
||
| type DynamicStructWithReader struct { | ||
| Instance interface{} | ||
| Reader dynamicstruct.Reader | ||
| Table Table | ||
| } | ||
|
|
||
| var validate *validator.Validate | ||
|
|
||
| func makeDynamicStructBuilder(fields map[string]Field) dynamicstruct.DynamicStruct { | ||
| custom_type := dynamicstruct.NewStruct() | ||
|
|
||
| var stringPointerType *string | ||
| var intPointerType *int | ||
| var floatPointerType *float32 // or 64 ? I don't see a good reason to use 64 | ||
| var boolPointerType *bool | ||
| var timePointerType *time.Time | ||
|
|
||
| // those fields are mandatory for all tables | ||
| custom_type.AddField("Object_id", stringPointerType, `validate:"required"`) | ||
| custom_type.AddField("Updated_at", timePointerType, `validate:"required"`) | ||
|
|
||
| for fieldName, field := range fields { | ||
| switch strings.ToLower(fieldName) { | ||
| case "object_id", "updated_at": | ||
| // already added above, with a different validation tag | ||
| break | ||
| default: | ||
| switch field.DataType { | ||
| case Bool: | ||
| custom_type.AddField(capitalize(fieldName), boolPointerType, "") | ||
| case Int: | ||
| custom_type.AddField(capitalize(fieldName), intPointerType, "") | ||
| case Float: | ||
| custom_type.AddField(capitalize(fieldName), floatPointerType, "") | ||
| case String: | ||
| custom_type.AddField(capitalize(fieldName), stringPointerType, "") | ||
| case Timestamp: | ||
| custom_type.AddField(capitalize(fieldName), timePointerType, "") | ||
| } | ||
| } | ||
| } | ||
| return custom_type.Build() | ||
| } | ||
|
|
||
| func validateParsedJson(instance interface{}) error { | ||
| validate = validator.New() | ||
| err := validate.Struct(instance) | ||
| if err != nil { | ||
|
|
||
| // This error should happen in the dynamic struct is badly formatted, or if the tags | ||
| // contain bad values. If this returns an error, it's a 500 error. | ||
| if _, ok := err.(*validator.InvalidValidationError); ok { | ||
| log.Println(err) | ||
| return err | ||
| } | ||
|
|
||
| // Otherwise it's a 400 error and we can access the reasons from here | ||
| count := 0 | ||
| for _, err := range err.(validator.ValidationErrors) { | ||
| fmt.Printf("The input object is not valid: key %v, validation tag: '%v', receive value %v", err.Field(), err.Tag(), err.Param()) | ||
| count++ | ||
| } | ||
| if count > 0 { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func ParseToDataModelObject(dataModel DataModel, jsonBody []byte, tableName string) (DynamicStructWithReader, error) { | ||
| table := dataModel.Tables[tableName] | ||
| fields := table.Fields | ||
|
|
||
| custom_type := makeDynamicStructBuilder(fields) | ||
|
|
||
| dynamicStructInstance := custom_type.New() | ||
| dynamicStructReader := dynamicstruct.NewReader(dynamicStructInstance) | ||
|
|
||
| // This is where errors can happen while parson the json. We could for instance have badly formatted | ||
| // json, or timestamps. | ||
| // We could also have more serious errors, like a non-capitalized field in the dynamic struct that | ||
| // causes a panic. We should manage the errors accordingly. | ||
| err := json.Unmarshal(jsonBody, &dynamicStructInstance) | ||
| if err != nil { | ||
| // add code here to distinguish between badly formatted json and other errors | ||
| return DynamicStructWithReader{Instance: dynamicStructInstance, Reader: dynamicStructReader, Table: table}, err | ||
| } | ||
|
|
||
| // If the data has been successfully parsed, we can validate it | ||
| // This is done using the validate tags on the dynamic struct | ||
| // There are two possible cases of error | ||
| err = validateParsedJson(dynamicStructInstance) | ||
| if err != nil { | ||
| return DynamicStructWithReader{Instance: dynamicStructInstance, Reader: dynamicStructReader, Table: table}, err | ||
| } | ||
|
|
||
| return DynamicStructWithReader{Instance: dynamicStructInstance, Reader: dynamicStructReader, Table: table}, nil | ||
| } | ||
|
|
||
| func ReadFieldFromDynamicStruct(dynamicStruct DynamicStructWithReader, fieldName string) interface{} { | ||
| check := dynamicStruct.Reader.HasField((capitalize(fieldName))) | ||
| if !check { | ||
| log.Fatalf("Field %v not found in dynamic struct", fieldName) | ||
| } | ||
| field := dynamicStruct.Reader.GetField(capitalize(fieldName)) | ||
| table := dynamicStruct.Table | ||
| fields := table.Fields | ||
| fieldFromModel, ok := fields[fieldName] | ||
| if !ok { | ||
| log.Fatalf("Field %v not found in table when reading from dynamic struct", fieldName) | ||
| } | ||
|
|
||
| switch fieldFromModel.DataType { | ||
| case Bool: | ||
| return field.PointerBool() | ||
| case Int: | ||
| return field.PointerInt() | ||
| case Float: | ||
| return field.PointerFloat32() | ||
| case String: | ||
| return field.PointerString() | ||
| case Timestamp: | ||
| return field.PointerTime() | ||
| default: | ||
| log.Fatal("Unknown data type") | ||
| return nil | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.