Draft reader/codec factory draft - WIP.

MAINTAINERS.md

@@ -3,14 +3,23 @@ Please keep the list sorted alphabetically by handle
 
 -----
 
-
 | Github ID                                    | Name                   | Email                              |
 |----------------------------------------------|------------------------|------------------------------------|
-| [@jacarey](https://github.com/jacarey)       | Jay Carey              | <jcarey@broadinstitute.org>        |
+| [@cmnbroad](https://github.com/cmnbroad)     | Chris Norman           | <cnorman@broadinstitute.org>       |
 | [@lbergelson](https://github.com/lbergelson) | Louis Bergelson        | <[email protected]>        |
 | [@magicDGS](https://github.com/magicDGS)     | Daniel Gómez Sánchez   | <[email protected]>  |
 | [@tfenne](https://github.com/tfenne)         | Tim Fennell            | <[email protected]>                |
 
 ----
 
+### These are prior project maintainers
+
+-----
+
+| Github ID                                    | Name                   | Email                              |
+|----------------------------------------------|------------------------|------------------------------------|
+| [@jacarey](https://github.com/jacarey)       | Jay Carey              | <[email protected]>        |
+
+----
+
 All of the current maintainers can be contacted by mentioning @samtools/htsjdk-next-maintainers

appveyor.yml

@@ -3,10 +3,6 @@ version: '{branch}.{build}'
 pull_requests:
   do_not_increment_build_number: true
 
-branches:
-  only:
-  - master
-
 environment:
   matrix:
   - JAVA_HOME: C:\Program Files\Java\jdk1.8.0

build.gradle

@@ -8,6 +8,8 @@ buildscript {
 plugins {
     id 'com.github.johnrengelman.shadow' version '2.0.4'
     id 'java-library'
+    id 'scala'
+    id 'com.github.maiflai.scalatest' version '0.21'
 }
 
 //This allows you to build a single shadowJar with the contents of all the
@@ -18,6 +20,8 @@ dependencies {
 
 subprojects {
     apply plugin: 'java-library'
+    apply plugin: 'scala'
+    apply plugin: 'com.github.maiflai.scalatest'
 
     group 'org.htsjdk'
     version '0.0.1'
@@ -33,13 +37,12 @@ subprojects {
         testCompile "com.google.jimfs:jimfs:1.1"
         testCompile "org.apache.commons:commons-lang3:3.7"
 
-        compile 'commons-io:commons-io:2.5'
-    }
+        testCompile "org.scala-lang:scala-library:2.12.4"
+        testCompile 'org.scalatest:scalatest_2.12:3.0.5'
+        testRuntime 'org.pegdown:pegdown:1.4.2'
 
-    test {
-        useTestNG{}
+        compile 'commons-io:commons-io:2.5'
     }
-
 }
 
 project(':cram') {

core/out/production/resources/META-INF/services/org.htsjdk.core.api.HtsjdkCodecDescriptor

@@ -0,0 +1,2 @@
+org.htsjdk.core.impl.bam.BAMCodecDescriptor
+

core/src/main/java/org/htsjdk/core/api/HtsjdkCodec.java

@@ -0,0 +1,13 @@
+package org.htsjdk.core.api;
+
+import org.htsjdk.core.api.io.IOResource;
+import org.htsjdk.core.utils.Version;
+
+import java.nio.file.Path;
+
+/**
+ * Base interface that must be implemented by all htsjdk codecs
+ */
+public interface HtsjdkCodec extends Upgradeable {
+
+}

core/src/main/java/org/htsjdk/core/api/HtsjdkCodecDescriptor.java

@@ -0,0 +1,39 @@
+package org.htsjdk.core.api;
+
+import org.htsjdk.core.api.io.IOResource;
+import org.htsjdk.core.utils.PathSpecifier;
+
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Base interface that must be implemented by all htsjdk codec descriptors. Descriptors
+ * are lightweight object that are cached by the registry service and used to locate
+ * and instantiate a codec for a given input.
+ */
+public interface HtsjdkCodecDescriptor {
+
+    String getName();
+
+    // Get the minimum number of bytes this codec needs in order to determine whether it can decode a stream.
+    int getMinimalFileSignatureSize();
+
+    boolean canDecode(final IOResource resource);
+
+    boolean canDecode(final Path path);
+
+    HtsjdkCodec getCodecInstance(final IOResource resource);
+
+    HtsjdkCodec getCodecInstance(final Path path);
+
+    HtsjdkCodec getCodecInstance(final InputStream is);
+
+    boolean canDecodeSignature(final byte[] streamSignature);
+
+    //Given a pathSpec, resolve any companion sibling files against it (index, .dict, etc.)
+    default List<PathSpecifier> resolveSiblings(final PathSpecifier pathSpec) {
+        return Collections.singletonList(pathSpec);
+    }
+}

core/src/main/java/org/htsjdk/core/api/Upgradeable.java

@@ -0,0 +1,9 @@
+package org.htsjdk.core.api;
+
+import org.htsjdk.core.utils.Version;
+
+public interface Upgradeable {
+
+    boolean runVersionUpgrade(final Version sourceVersion, final Version targetVersion);
+
+}

core/src/main/java/org/htsjdk/core/exception/HtsjdkPluginException.java

@@ -0,0 +1,16 @@
+package org.htsjdk.core.exception;
+
+/**
+ * Base class for exceptions resulting from ill-behaved codec plugins
+ */
+public class HtsjdkPluginException extends HtsjdkException {
+    /**
+     * Constructs an HTSJDK exception.
+     *
+     * @param message detailed message.
+     */
+    public HtsjdkPluginException(String message) {
+        super(message);
+    }
+
+}

core/src/main/java/org/htsjdk/core/impl/bam/BAMCodec.java

@@ -0,0 +1,44 @@
+package org.htsjdk.core.impl.bam;
+
+import org.htsjdk.core.api.HtsjdkCodec;
+import org.htsjdk.core.api.io.IOResource;
+import org.htsjdk.core.utils.Version;
+
+import java.io.InputStream;
+import java.nio.file.Path;
+
+/**
+ * Mock, do-nothing BAM codec used to exercise the reader factory infrastructure
+ */
+public class BAMCodec implements HtsjdkCodec {
+
+    public BAMCodec(final IOResource ioResource) { this(ioResource.toPath()); }
+
+    public BAMCodec(final Path pathResource) {
+        // TODO
+    }
+
+    public BAMCodec(final InputStream inputStream) {
+
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return super.equals(obj);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString();
+    }
+
+    @Override
+    public boolean runVersionUpgrade(final Version sourceVersion, final Version targetVersion) {
+        return false;
+    }
+}

core/src/main/java/org/htsjdk/core/impl/bam/BAMCodecDescriptor.java

@@ -0,0 +1,82 @@
+package org.htsjdk.core.impl.bam;
+
+import org.htsjdk.core.api.HtsjdkCodecDescriptor;
+import org.htsjdk.core.api.HtsjdkCodec;
+import org.htsjdk.core.api.io.IOResource;
+import org.htsjdk.core.exception.HtsjdkIOException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Mock, do-nothing BAM codec descriptor.
+ */
+public class BAMCodecDescriptor implements HtsjdkCodecDescriptor {
+
+    public static final String BAM_FILE_EXTENSION = ".bam";
+    public static final String BAM_MAGIC = "BAM\1";
+
+    @Override
+    public String getName() {
+        return "BAM codec descriptor";
+    }
+
+    @Override
+    public int getMinimalFileSignatureSize() {
+        return BAM_MAGIC.length();
+    }
+
+    @Override
+    public boolean canDecode(final IOResource resource) {
+        return resource.getURIString().endsWith(BAM_FILE_EXTENSION);
+    }
+
+    @Override
+    public boolean canDecode(final Path path) {
+        return path.endsWith(BAM_FILE_EXTENSION);
+    }
+
+    // uses a byte array rather than a stream to reduce the need to repeatedly mark/reset the
+    // stream for each codec
+    @Override
+    public boolean canDecodeSignature(final byte[] signatureBytes) {
+        return signatureBytes.equals("BAM");
+    }
+
+    @Override
+    public HtsjdkCodec getCodecInstance(final IOResource ioResource) {
+        return getCodecInstance(ioResource.toPath());
+    }
+
+    @Override
+    public HtsjdkCodec getCodecInstance(final Path resourcePath) {
+        try {
+            return new BAMCodec(Files.newInputStream(resourcePath));
+        } catch (IOException e) {
+            throw new HtsjdkIOException(e);
+        }
+    }
+
+    @Override
+    public HtsjdkCodec getCodecInstance(final InputStream is) {
+        return new BAMCodec(is);
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return super.equals(obj);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString();
+    }
+
+}

core/src/main/java/org/htsjdk/core/utils/CodecDiscoveryService.java

@@ -0,0 +1,24 @@
+package org.htsjdk.core.utils;
+
+import org.htsjdk.core.api.HtsjdkCodecDescriptor;
+
+import java.util.*;
+
+/**
+ * Service loader for dynamically discovering htsjdk codecs.
+ */
+public class CodecDiscoveryService {
+    private static ServiceLoader<HtsjdkCodecDescriptor> serviceLoader = ServiceLoader.load(HtsjdkCodecDescriptor.class);
+
+    private CodecDiscoveryService() {}
+
+    public static List<HtsjdkCodecDescriptor> discoverCodecs() {
+        final List<HtsjdkCodecDescriptor> descriptors = new ArrayList<>();
+
+        final Iterator<HtsjdkCodecDescriptor> descriptorIterator = serviceLoader.iterator();
+        while (descriptorIterator.hasNext()) {
+            descriptors.add(descriptorIterator.next());
+        }
+        return descriptors;
+    }
+}

core/src/main/java/org/htsjdk/core/utils/CodecRegistry.java

@@ -0,0 +1,83 @@
+package org.htsjdk.core.utils;
+
+import org.htsjdk.core.api.HtsjdkCodec;
+import org.htsjdk.core.api.HtsjdkCodecDescriptor;
+import org.htsjdk.core.api.io.IOResource;
+import org.htsjdk.core.exception.HtsjdkIOException;
+import org.htsjdk.core.exception.HtsjdkPluginException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+
+/**
+ * Registry/cache for codec descriptors for discovered codecs.
+ */
+public class CodecRegistry {
+    private static final CodecRegistry codecRegistry = new CodecRegistry();
+
+    private static List<HtsjdkCodecDescriptor> discoveredCodecs = new ArrayList<>();
+
+    // minimum number of bytes required to allow any codec to deterministically decide if it can
+    // decode a stream
+    private static int minSignatureSize = 0;
+
+    static {
+        CodecDiscoveryService.discoverCodecs().forEach(codecRegistry::addCodecDescriptor);
+    }
+
+    /**
+     * Add a codec to the registry
+     */
+    private void addCodecDescriptor(final HtsjdkCodecDescriptor codecDescriptor) {
+        discoveredCodecs.add(codecDescriptor);
+        final int minSignatureBytesRequired = codecDescriptor.getMinimalFileSignatureSize();
+        if (minSignatureBytesRequired < 1) {
+            throw new HtsjdkPluginException(
+                    String.format("%s: getMinimalFileSignatureSize must be > 0", codecDescriptor.getName())
+            );
+        }
+        minSignatureSize = Integer.max(minSignatureSize, minSignatureBytesRequired);
+    }
+
+    // TODO: this should have a name and contract that reflects that its only looking at the URI
+    // Once we find a codec, hand it off already primed with the version header, etc).
+    public static HtsjdkCodec findCodecFor(final IOResource inputResource) {
+        final Optional<HtsjdkCodecDescriptor> codec =
+                discoveredCodecs.stream()
+                    .filter(codecDescriptor -> codecDescriptor.canDecode(inputResource))
+                    .findFirst();
+
+        if (codec.isPresent()) {
+            return codec.get().getCodecInstance(inputResource);
+        } else {
+            //TODO: who closes/owns the lifetime of this input stream ?
+            InputStream is = inputResource.getInputStream();
+            return findCodecFor(inputResource.toString(), is);
+        }
+    }
+
+    // Once we find a codec, hand it off already primed with the version header, etc).
+    public static HtsjdkCodec findCodecFor(final String sourceName, final InputStream is) {
+        final byte[] signatureBytes = new byte[minSignatureSize];
+        try {
+            final int numRead = is.read(signatureBytes);
+            if (numRead <= 0) {
+                throw new HtsjdkIOException(String.format("Failure reading content from stream for %s", sourceName));
+            }
+            return discoveredCodecs.stream()
+                    .filter(
+                        // its possible that the input is a legitimate stream for some codec, but
+                        // contains less bytes than are required even for signature detection by another
+                        // codecs, so skip any descriptors that require more bytes than are available
+                        codecDescriptor ->
+                                numRead >= codecDescriptor.getMinimalFileSignatureSize() &&
+                                codecDescriptor.canDecodeSignature(signatureBytes))
+                    .findFirst()
+                    .orElseThrow(() -> new HtsjdkIOException(String.format("No codec found for %s", sourceName)))
+                    .getCodecInstance(is);
+        } catch (IOException e) {
+            throw new HtsjdkIOException(String.format("Failure reading signature from stream for %s", sourceName), e);
+        }
+    }
+}