Merge pull request #195 from RADAR-base/release-0.5.4

Release 0.5.4

.dockerignore

@@ -1,6 +1,6 @@
 java-sdk/**/build
 **/.idea
-java-sdk/**/.gradle
+java-sdk/.gradle
 java-sdk/**/out
 java-sdk/**/src/test
 .git

Dockerfile.kafkaInit

@@ -0,0 +1,55 @@
+FROM openjdk:11-jdk-slim as builder
+
+RUN mkdir -p /code/java-sdk
+WORKDIR /code/java-sdk
+
+COPY ./commons /code/commons
+COPY ./specifications /code/specifications
+COPY ./java-sdk/gradle /code/java-sdk/gradle
+COPY ./java-sdk/build.gradle /code/java-sdk/build.gradle
+COPY ./java-sdk/settings.gradle java-sdk/gradlew /code/java-sdk/
+
+ENV GRADLE_OPTS -Dorg.gradle.daemon=false
+
+RUN ./gradlew tasks
+COPY ./java-sdk/radar-schemas-commons/build.gradle /code/java-sdk/radar-schemas-commons/
+COPY ./java-sdk/radar-schemas-commons/src /code/java-sdk/radar-schemas-commons/src
+RUN ./gradlew :radar-schemas-commons:jar
+COPY ./java-sdk/radar-schemas-tools/build.gradle /code/java-sdk/radar-schemas-tools/
+COPY ./java-sdk/radar-schemas-tools/src /code/java-sdk/radar-schemas-tools/src
+RUN ./gradlew distTar && cd radar-schemas-tools/build/distributions && tar xzf radar-schemas-tools*.tar.gz
+
+
+FROM openjdk:11-jdk-slim
+
+ENV KAFKA_SCHEMA_REGISTRY=http://schema-registry-1:8081
+ENV KAFKA_NUM_PARTITIONS=3
+ENV KAFKA_NUM_REPLICATION=3
+ENV KAFKA_NUM_BROKERS=3
+ENV KAFKA_ZOOKEEPER_CONNECT=zookeeper-1:2181
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		curl \
+		rsync \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /schema/merged /schema/java/src /schema/java/classes /usr/share/java
+
+WORKDIR /schema
+
+RUN curl -#o /usr/share/java/avro-tools.jar \
+ "http://archive.apache.org/dist/avro/avro-1.8.2/java/avro-tools-1.8.2.jar"
+
+RUN mkdir original/
+
+COPY --from=builder /code/java-sdk/radar-schemas-tools/build/distributions/radar-schemas-tools-*/lib/* /usr/lib/
+COPY --from=builder /code/java-sdk/radar-schemas-tools/build/distributions/radar-schemas-tools-*/bin/radar-schemas-tools /usr/bin/
+COPY --from=builder /code ./original
+
+VOLUME /schema/conf
+# Copy bash file
+COPY ./docker/topic_init.sh ./docker/init.sh ./docker/list_aggregated.sh ./docker/list_raw.sh /usr/bin/
+RUN chmod +x /usr/bin/*.sh
+
+ENTRYPOINT ["init.sh"]
+CMD ["topic_init.sh"]

commons/connector/upload/altoida/altoida_action.avsc

@@ -2,11 +2,11 @@
   "namespace": "org.radarcns.connector.upload.altoida",
   "type": "record",
   "name": "AltoidaAction",
-  "doc": "Logs of actions taken.",
+  "doc": "User action taken during a test with the Altoida app.",
   "fields": [
     {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
     {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},
-    {"name": "action", "type": "string", "doc": "Log of actions taken during test."},
+    {"name": "action", "type": "string", "doc": "Action taken by the user during test."},
     {"name": "extraInfo", "type": "string", "doc": "Extra information about action taken."}
   ]
 }

commons/connector/upload/altoida/altoida_blink.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "type": "record",
+  "name": "AltoidaBlink",
+  "doc": "Eye blinking event data of the Aldoida app user. Possible values are LEFT, RIGHT, BOTH, DETECTED.",
+  "fields": [
+    {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
+    {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},
+    {"name": "eye", "type": "string", "doc": "Which of the eyes blinked. LEFT, RIGHT, BOTH, DETECTED."}
+  ]
+}

commons/connector/upload/altoida/altoida_diagnostics.avsc

@@ -2,7 +2,7 @@
    "namespace": "org.radarcns.connector.upload.altoida",
    "type": "record",
    "name": "AltoidaDiagnostics",
-   "doc": "Data indicating what object was placed or found.",
+   "doc": "What object was placed or found in an Altoida trial.",
    "fields": [
      {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
      {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},

commons/connector/upload/altoida/altoida_domain_result.avsc

@@ -0,0 +1,19 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "type": "record",
+  "name": "AltoidaDomainResult",
+  "doc": "Nine domain percentile values 0 to 100 as described below (percent of healthy users that perform worse in the respective domain).",
+  "fields": [
+    {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
+    {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},
+    {"name": "perceptualMotorCoordination", "type": "float", "doc": "Percentage value of perceptual motor coordination."},
+    {"name": "complexAttention", "type": "float", "doc": "Percentage value of complex attention."},
+    {"name": "cognitiveProcessingSpeed", "type": "float", "doc": "Percentage value of cognitive processing speed."},
+    {"name": "inhibition", "type": "float", "doc": "Percentage value of inhibition."},
+    {"name": "flexibility", "type": "float", "doc": "Percentage value of flexibility."},
+    {"name": "visualPerception", "type": "float", "doc": "Percentage value of visual perception."},
+    {"name": "planning", "type": "float", "doc": "Percentage value of planning."},
+    {"name": "prospectiveMemory", "type": "float", "doc": "Percentage value of prospective memory."},
+    {"name": "spatialMemory", "type": "float", "doc": "Percentage value of spatial memory."}
+  ]
+}

commons/connector/upload/altoida/altoida_eye_tracking.avsc

@@ -0,0 +1,12 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "type": "record",
+  "name": "AltoidaEyeTracking",
+  "doc": "Position where users eye movement on the screen is detected. These values are calculated from the center of the screen in centimeters (cm).",
+  "fields": [
+    {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
+    {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},
+    {"name": "x", "type": ["null", "float"], "doc": "X coordinate of the eye position.", "default": null},
+    {"name": "y", "type": ["null", "float"], "doc": "Y coordinate of the eye position.", "default": null}
+  ]
+}

commons/connector/upload/altoida/altoida_magnetometer.avsc → commons/connector/upload/altoida/altoida_magnetic_field.avsc

@@ -1,7 +1,7 @@
 {
   "namespace": "org.radarcns.connector.upload.altoida",
   "type": "record",
-  "name": "AltoidaMagnetometer",
+  "name": "AltoidaMagneticField",
   "doc": "Data from 3-axis magnetometer sensor micro Tesla units (µT).",
   "fields": [
     {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},

commons/connector/upload/altoida/altoida_summary.avsc

@@ -0,0 +1,17 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "type": "record",
+  "name": "AltoidaSummary",
+  "doc": "Summary and aggregates of a full test with Altoida. Original file is named as export.csv.",
+  "fields": [
+    {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
+    {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},
+    {"name": "label", "type": "string", "doc": "Label name of session as entered in the app."},
+    {"name": "age", "type": "int", "doc": "Age of the participant in years."},
+    {"name": "yearsOfEducation", "type": "int", "doc": "Years of education in years."},
+    {"name": "gender", "type": "int", "doc": "Gender of the participant 0=male, 1=female, 2=other."},
+    {"name": "class", "type": "int", "doc": "Classification based on the analysis  0=healthy, 1=at risk, 2=MCI due to AD."},
+    {"name": "nmi", "type": "double", "doc": "NMI value of the test,a value in 0-100, where 0-33.3 is class 2, 33.3-66.6 is class 1, 66.6-100 is class 0."},
+    {"name": "groundTruth", "type": "int", "doc": "If the test is based on additional biomarkers -1=unknown, 0=healthy, 1= at risk, 2=MCI/AB+, 3=MCI/AB-."}
+  ]
+}

commons/connector/upload/altoida/altoida_summary_metrics.avsc

@@ -0,0 +1,49 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "name": "AltoidaSummaryMetrics",
+  "type": "record",
+  "doc": "Metrics from a set of tests.",
+  "fields": [
+    {"name": "audioHighReactionTimes", "type": "float", "doc": "Distribution of reaction times to high tone(s)."},
+    {"name": "audioHighAccuracy", "type": "float", "doc": "Distribution of deviation from speaker button center (cm)."},
+    {"name": "audioLowReactions", "type": "int", "doc": "Distributions of number of reactions to low tones."},
+    {"name": "audioIgnoredHighTonePercentage", "type": "float", "doc": "Distribution of ratio of high tones ignored."},
+    {"name": "audioPrematureToneButtonPresses", "type": "int", "doc": "Distribution of number of premature tone button presses."},
+    {"name": "randomScreenPressesDuringPlacement", "type": "int", "doc": "Distribution of number of random screen presses during the placement phase."},
+    {"name": "randomScreenPressesDuringSearch", "type": "int", "doc": "Distribution of number of random screen presses during placement."},
+    {"name": "tooMuchMovementCount", "type": "int", "doc": "Distribution of number of times placement was impossible due to too much movement."},
+    {"name": "findBetterPlaceCount", "type": "float", "doc": "Number of times the user had to find a better place."},
+    {"name": "introReadTimes", "type": "float", "doc": "Mean time the user required to read an intro page (s)."},
+    {"name": "introReadTime1", "type": "float", "doc": "Time the user required to read the first intro page (s)."},
+    {"name": "introReadTime2", "type": "float", "doc": "Time the user required to read the second intro page (s)."},
+    {"name": "placeDelays", "type": "float", "doc": "Mean delay from possible placement to actual placement (s)."},
+    {"name": "spotAlreadyTakenCount", "type": "int", "doc": "Number of times the user tried to use the same spot twice."},
+    {"name": "trials", "type": {
+      "type": "array",
+      "items": "AltoidaTrial"
+    }, "doc": "Per-trials details and analyses."},
+    {"name": "trialMeans", "type": "AltoidaTrial", "doc": "Mean values of each of the trials."},
+    {"name": "findFailCount", "type": "float", "doc": "Number of times the user failed to find an object."},
+    {"name": "findSkipDurations", "type": "float", "doc": "Mean time the user tried before skipping an object in seconds."},
+    {"name": "skipButtonCount", "type": "float", "doc": "Number of times the user pressed the skip button."},
+    {"name": "countdownFail", "type": "float", "doc": "Number of countdown fails."},
+    {"name": "stepCountRatio", "type": "float", "doc": "Distribution of the ratio between the steps needed for finding the objects over the steps needed for placing them."},
+    {"name": "stepDelayP", "type": "float", "doc": "Mean delay between steps when placing objects (s)."},
+    {"name": "stepDelayF", "type": "float", "doc": "Mean delay between steps when finding objects (s)."},
+    {"name": "stepVarianceP", "type": "float", "doc": "Variance of delays between steps when placing objects."},
+    {"name": "stepVarianceF", "type": "float", "doc": "Variance of delays between steps when finding objects."},
+    {"name": "notWalkingTimeP", "type": "float", "doc": "Time the user did not walk during placing objects (s)."},
+    {"name": "notWalkingTimeF", "type": "float", "doc": "Time the user did not walk during finding objects (s)."},
+    {"name": "shockCount", "type": "float", "doc": "Number of shocks that were recorded during the test."},
+    {"name": "accelerationVarianceX", "type": "float", "doc": "Variance of acceleration data of x axis."},
+    {"name": "accelerationVarianceY", "type": "float", "doc": "Variance of acceleration data of y axis."},
+    {"name": "accelerationVarianceZ", "type": "float", "doc": "Variance of acceleration data of z axis."},
+    {"name": "strongHand", "type": "float", "doc": "Dominant hand of user."},
+    {"name": "normalisationCircle", "type": "AltoidaWalkingTestAggregate", "doc": "Deviation from optimal path in circle normalisation test (cm)."},
+    {"name": "normalisationSquare", "type": "AltoidaWalkingTestAggregate", "doc": "Deviation from optimal path in square normalisation test (cm)."},
+    {"name": "normalisationSerpentine", "type": "AltoidaWalkingTestAggregate", "doc": "Deviation from optimal path in serpentine normalisation test (cm)."},
+    {"name": "normalisationSpeedCircle", "type": "AltoidaWalkingTestAggregate", "doc": "Deviation from optimal path in speed circle normalisation test (cm)."},
+    {"name": "normalisationRandomTapping", "type": "AltoidaTappingTestAggregate", "doc": "Deviation from center points in random tapping normalisation test (cm)."},
+    {"name": "normalisationTapping", "type": "AltoidaTappingTestAggregate", "doc": "Deviation from center points in tapping normalisation test (cm)."}          
+  ]
+}

commons/connector/upload/altoida/altoida_touchscreen.avsc → commons/connector/upload/altoida/altoida_tap.avsc

@@ -1,8 +1,8 @@
 {
   "namespace": "org.radarcns.connector.upload.altoida",
   "type": "record",
-  "name": "AltoidaTouchscreen",
-  "doc": "Data of location where the user touched the touchscreen on an x and y axis. These values are calculated from the center of the screen in centimeters (cm).",
+  "name": "AltoidaTap",
+  "doc": "Location where the user tapped the touchscreen on an x and y axis. These values are calculated from the center of the screen in centimeters (cm).",
   "fields": [
     {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
     {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},

commons/connector/upload/altoida/altoida_tapping_test_aggregate.avsc

@@ -0,0 +1,15 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "name": "AltoidaTappingTestAggregate",
+  "type": "record",
+  "doc": "Aggregate for a test on tapping the screen.",
+  "fields": [
+    { "name": "mean", "type": "float", "doc": "Mean test value."},
+    { "name": "variance", "type": "float", "doc": "Variance of test values."},
+    { "name": "count", "type": "int", "doc": "Number of tests performed."},
+    { "name": "reactionTime", "type": "float", "doc": "Mean time to react by the user (s)."},
+    { "name": "rate", "type": "float", "doc": "Number of tests performed per second."},
+    { "name": "duration", "type": "float", "doc": "Mean duration of a test (s)."},
+    { "name": "fullDuration", "type": "float", "doc": "Total amount of time performing tests (s)."}
+  ]
+}

commons/connector/upload/altoida/altoida_touch.avsc

@@ -0,0 +1,16 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "type": "record",
+  "name": "AltoidaTouch",
+  "doc": "Location where the user touched the touchscreen on an x and y axis. These values are calculated from the center of the screen in centimeters (cm).",
+  "fields": [
+    {"name": "time", "type": "double", "doc": "Timestamp in unix time acquired locally from the device."},
+    {"name": "timeReceived", "type": "double", "doc": "Timestamp in unix time received from Altoida API."},
+    {"name": "x", "type": "double", "doc": "Distance from the center of the screen in (cm)s on x-axis."},
+    {"name": "y", "type": "double", "doc": "Distance from the center of the screen in (cm)s on y-axis."},
+    {"name": "surface", "type": "double", "doc": "Diameter of the touch area in cm."},
+    {"name": "accelerometer", "type": "double", "doc": "Acceleration exerted by the finger hitting the screen."},
+    {"name": "combined", "type": "double", "doc": "An overall value of touch pressure formed from a linear combination of the previous columns."},
+    {"name": "hit", "type": "boolean", "doc": "Whether the touch hit a button or not, 1 if the touch served a purpose basically, 0 otherwise."}
+  ]
+}

commons/connector/upload/altoida/altoida_trial.avsc

@@ -0,0 +1,13 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "name": "AltoidaTrial",
+  "type": "record",
+  "doc": "Trial statistics.",
+  "fields": [
+    {"name": "placeDuration", "type": "float", "doc": "Time the user required to place the object (s)."},
+    {"name": "findDuration", "type": "float", "doc": "Time the user required to find the object (s)."},
+    {"name": "stepAngularChange", "type": "float", "doc": "Mean angle the user turned during one step when finding the object (s)."},
+    {"name": "totalAngularChange", "type": "float", "doc": "Accumulated angular change when finding the object (radians)."},
+    {"name": "pathComplexity", "type": "float", "doc": "Path complexity when finding the object (m)."}
+  ]
+}

commons/connector/upload/altoida/altoida_walking_test_aggregate.avsc

@@ -0,0 +1,21 @@
+{
+  "namespace": "org.radarcns.connector.upload.altoida",
+  "name": "AltoidaWalkingTestAggregate",
+  "type": "record",
+  "doc": "Aggregate data of a walking test.",
+  "fields": [
+    { "name": "mean", "type": "float", "doc": "Mean test value."},
+    { "name": "variance", "type": "float", "doc": "Variance of test values."},
+    { "name": "timeRatioWithin", "type": "float", "doc": "Ratio of time spent within the specified area."},
+    { "name": "distRatioWithin", "type": "float", "doc": "Ratio of distance spent within the specified area."},
+    { "name": "distWithin", "type": "float", "doc": "Distance spent within the specified area (cm)."},
+    { "name": "distTotal", "type": "float", "doc": "Total distance moved in test (cm)."},
+    { "name": "angleSum", "type": ["null", "float"], "doc": "Sum of angular movement (radians). Null if not applicable.", "default": null},
+    { "name": "directionalChanges", "type": ["null", "int"], "doc": "Number of times the movement changed direction. Null if not applicable.", "default": null},
+    { "name": "verticalProgress", "type": ["null", "float"], "doc": "Vertical ratio of the serpentine that was completed. Null if not applicable.", "default": null},
+    { "name": "speed", "type": "float", "doc": "Average speed of movement (cm/s)."},
+    { "name": "speedAccuracyRatio", "type": "float", "doc": "Speed accuracy ratio (s)."},
+    { "name": "duration", "type": "float", "doc": "Mean duration of a test (s)."},
+    { "name": "fullDuration", "type": "float", "doc": "Total amount of time performing tests (s)."}
+  ]
+}

docker/init.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+AVRO_TOOLS=/usr/share/java/avro-tools.jar
+
+rsync -a /schema/original/commons /schema/original/specifications /schema/merged
+rsync -a /schema/conf/ /schema/merged
+
+# Compiling updated schemas
+echo "Compiling schemas..." >&2
+
+# Regex for schemas with a dependency that is a class
+# e.g., a literal class starting with a capital, or
+# a namespace with internal periods.
+DEPENDENT_REGEX='"(items|type)": (\[\s*"null",\s*)?"([A-Z]|[^".]*\.)'
+
+# Separate enums so that they can be referenced in later files
+find merged/commons -name "*.avsc" -print | sort > merged/file_list
+DEPENDENT=$(find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" "{}" \; -print | sort)
+# remove dependent files from all files to get independent files
+INDEPENDENT=$(printf "${DEPENDENT}" | comm -23 merged/file_list -)
+
+printf "===> Independent schemas:\n${INDEPENDENT}\n"
+printf "===> Dependent schemas:\n${DEPENDENT}\n"
+
+java -jar "${AVRO_TOOLS}" compile -string schema ${INDEPENDENT} ${DEPENDENT} java/src 2>/dev/null
+find java/src -name "*.java" -print0 | xargs -0 javac -cp /usr/lib/*:java/classes -d java/classes -sourcepath java/src 
+# Update the radar schemas so the tools find the new classes in classpath
+jar uf /usr/lib/radar-schemas-commons-*.jar -C java/classes .
+
+exec "$@"

docker/list_aggregated.sh

@@ -0,0 +1,22 @@
+#/bin/bash
+
+SAVEIFS=$IFS
+# Change IFS to new line.
+raw_str=$(radar-schemas-tools list -q --raw merged)
+all_str=$(radar-schemas-tools list -q merged)
+IFS=$'\n'
+raw=($raw_str)
+all=($all_str)
+# Restore IFS
+IFS=$SAVEIFS
+
+aggregated=()
+for i in "${all[@]}"; do
+   skip=
+   for j in "${raw[@]}"; do
+       [[ $i == $j ]] && { skip=1; break; }
+   done
+   [[ -n $skip ]] || aggregated+=("$i")
+done
+IFS=','
+echo "${aggregated[*]}"

docker/list_raw.sh

@@ -0,0 +1,2 @@
+#/bin/bash
+radar-schemas-tools list -q --raw merged | tr '\n' ','