Backport "Block common passwords" rule to Zimbra 9

.gitignore

@@ -12,3 +12,4 @@ store/.pmd
 store/.ruleset
 eclipse-ivysettings.xml
 store/tmp/
+common-passwords.txt

build.xml

@@ -6,6 +6,7 @@
       <ant dir="./store"  target="war"                inheritAll="true"/>
       <ant dir="./store"  target="create-version-sql" inheritAll="true"/>
       <ant dir="./store"  target="create-version-ldap" inheritAll="true"/>
+      <ant dir="./store-conf"  target="common-passwords" inheritAll="true"/>
     </target>
 
    <target name="publish-local-all">

common/src/java-test/com/zimbra/common/util/UnmodifiableBloomFilterTest.java

@@ -0,0 +1,124 @@
+/*
+ * ***** BEGIN LICENSE BLOCK *****
+ * Zimbra Collaboration Suite Server
+ * Copyright (C) 2019 Synacor, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software Foundation,
+ * version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License along with this program.
+ * If not, see <https://www.gnu.org/licenses/>.
+ * ***** END LICENSE BLOCK *****
+ */
+package com.zimbra.common.util;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class UnmodifiableBloomFilterTest {
+
+    protected UnmodifiableBloomFilter<String> bloomFilter = UnmodifiableBloomFilter
+        .createFilterFromFile("src/java-test/common-passwords.txt");
+
+    @Before
+    public void setUp() {
+        assertTrue(bloomFilter.isInitialized());
+        assertFalse(bloomFilter.isDisabled());
+    }
+
+    @Test
+    public void testMightContain() {
+        assertTrue(bloomFilter.isInitialized());
+        assertTrue(bloomFilter.mightContain("test123"));
+        assertTrue(bloomFilter.mightContain("hunter2"));
+    }
+
+    @Test
+    public void testMightContainFalse() {
+        assertTrue(bloomFilter.isInitialized());
+        assertFalse(bloomFilter.mightContain("not-in-the-test-file"));
+    }
+
+    @Test
+    public void testCreateFilterFromMissingFile() {
+        UnmodifiableBloomFilter<String> missingFileFilter = UnmodifiableBloomFilter
+            .createFilterFromFile("src/java-test/fake-file-not-found");
+        // expect to immediately initialize
+        assertTrue(missingFileFilter.isInitialized());
+        assertTrue(missingFileFilter.isDisabled());
+        assertFalse(missingFileFilter.mightContain("test123"));
+    }
+
+    @Test
+    public void testCreateFilterFromEmptySpecifiedFile() {
+        UnmodifiableBloomFilter<String> noFileFilter = UnmodifiableBloomFilter
+            .createFilterFromFile("");
+        // expect to immediately consider empty file as initialized
+        assertTrue(noFileFilter.isInitialized());
+        assertTrue(noFileFilter.isDisabled());
+        assertFalse(noFileFilter.mightContain("test123"));
+    }
+
+    @Test
+    public void testCreateFilterFromNullSpecifiedFile() {
+        UnmodifiableBloomFilter<String> noFileFilter = UnmodifiableBloomFilter
+            .createFilterFromFile(null);
+        // expect to immediately consider null file as initialized
+        assertTrue(noFileFilter.isInitialized());
+        assertTrue(noFileFilter.isDisabled());
+        assertFalse(noFileFilter.mightContain("test123"));
+    }
+
+    @Test
+    public void testMightContainLazyLoad() {
+        UnmodifiableBloomFilter<String> lazyFilter = UnmodifiableBloomFilter
+            .createLazyFilterFromFile("src/java-test/common-passwords.txt");
+        // expect to initialize on demand
+        assertFalse(lazyFilter.isInitialized());
+        assertFalse(lazyFilter.isDisabled());
+        assertTrue(lazyFilter.mightContain("test123"));
+        assertTrue(lazyFilter.mightContain("hunter2"));
+        assertTrue(lazyFilter.isInitialized());
+        assertFalse(lazyFilter.isDisabled());
+    }
+
+    @Test
+    public void testCreateLazyFilterFromMissingFile() {
+        UnmodifiableBloomFilter<String> missingFileFilter = UnmodifiableBloomFilter
+            .createLazyFilterFromFile("src/java-test/fake-file-not-found");
+        // expect to initialize on demand
+        assertFalse(missingFileFilter.isInitialized());
+        assertFalse(missingFileFilter.mightContain("test123"));
+        assertTrue(missingFileFilter.isInitialized());
+        // file not found results in disabled instance
+        assertTrue(missingFileFilter.isDisabled());
+    }
+
+    @Test
+    public void testCreateLazyFilterFromEmptySpecifiedFile() {
+        UnmodifiableBloomFilter<String> noFileFilter = UnmodifiableBloomFilter
+            .createLazyFilterFromFile("");
+        // expect to immediately consider empty file as initialized
+        assertTrue(noFileFilter.isInitialized());
+        assertTrue(noFileFilter.isDisabled());
+        assertFalse(noFileFilter.mightContain("test123"));
+    }
+
+    @Test
+    public void testCreateLazyFilterFromNullSpecifiedFile() {
+        UnmodifiableBloomFilter<String> noFileFilter = UnmodifiableBloomFilter
+            .createLazyFilterFromFile(null);
+        // expect to immediately consider null file as initialized
+        assertTrue(noFileFilter.isInitialized());
+        assertTrue(noFileFilter.isDisabled());
+        assertFalse(noFileFilter.mightContain("test123"));
+    }
+
+}

common/src/java-test/common-passwords.txt

@@ -0,0 +1,14 @@
+test123
+bad-pass
+hunter2
+abc123
+football
+monkey
+letmein
+696969
+shadow
+master
+666666
+qwertyuiop
+123321
+mustang

common/src/java/com/zimbra/common/localconfig/LC.java

@@ -1436,6 +1436,12 @@ public enum PUBLIC_SHARE_VISIBILITY { samePrimaryDomain, all, none };
     // to switch to Tika com.zimbra.cs.convert.TikaExtractionClient
     public static final KnownKey attachment_extraction_client_class = KnownKey.newKey("com.zimbra.cs.convert.LegacyConverterClient");
 
+    // list file for blocking common passwords
+    public static final KnownKey common_passwords_txt = KnownKey.newKey("${zimbra_home}/conf/common-passwords.txt");
+
+    // enable blocking common passwords
+    public static final KnownKey zimbra_block_common_passwords_enabled = KnownKey.newKey(false);
+
     static {
         // Automatically set the key name with the variable name.
         for (Field field : LC.class.getFields()) {

common/src/java/com/zimbra/common/util/UnmodifiableBloomFilter.java

@@ -0,0 +1,163 @@
+/*
+ * ***** BEGIN LICENSE BLOCK *****
+ * Zimbra Collaboration Suite Server
+ * Copyright (C) 2019 Synacor, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software Foundation,
+ * version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License along with this program.
+ * If not, see <https://www.gnu.org/licenses/>.
+ * ***** END LICENSE BLOCK *****
+ */
+package com.zimbra.common.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import com.google.common.hash.BloomFilter;
+import com.google.common.hash.Funnels;
+
+public class UnmodifiableBloomFilter<E> {
+
+    private static final Double DEFAULT_TOLERANCE = 0.03;
+    private static final Integer DEFAULT_BUFFER_SIZE = 4096;
+    private static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+
+    private BloomFilter<E> cache;
+    private boolean isInitialized = false;
+    private double tolerance;
+    private final String fileLocation;
+
+    private UnmodifiableBloomFilter(String fileLocation, BloomFilter<E> cache, double tolerance) {
+        this.fileLocation = fileLocation;
+        this.cache = cache;
+        this.tolerance = tolerance;
+        this.isInitialized = StringUtil.isNullOrEmpty(fileLocation) ? true : false;
+    }
+
+    private UnmodifiableBloomFilter(BloomFilter<E> cache) {
+        this.fileLocation = null;
+        this.cache = cache;
+        this.isInitialized = true;
+    }
+
+    public boolean mightContain(E entry) {
+        ensureInitialized();
+        return cache != null && cache.mightContain(entry);
+    }
+
+    /**
+     * @return True if an attempt to load/set the inner cache was made
+     */
+    public boolean isInitialized() {
+        return isInitialized;
+    }
+
+    /**
+     * @return True if the instance is initialized but still null.
+     */
+    public boolean isDisabled() {
+        return isInitialized && cache == null;
+    }
+
+    @SuppressWarnings("unchecked")
+    private void ensureInitialized() {
+        if (isInitialized) {
+            return;
+        }
+        synchronized (fileLocation) {
+            if (isInitialized) {
+                return;
+            }
+            this.cache = (BloomFilter<E>) loadFilter(new File(fileLocation), tolerance);
+            isInitialized = true;
+        }
+    }
+
+    /**
+     * Wraps a bloom filter so it may not be modified.
+     * @param filter The filter to wrap
+     * @return Unmodifiable bloom filter
+     */
+    public static <E> UnmodifiableBloomFilter<E> createFilter(BloomFilter<E> filter) {
+        return new UnmodifiableBloomFilter<E>(filter);
+    }
+
+    /**
+     * @see UnmodifiableBloomFilter#createFilterFromFile(String, double)
+     */
+    public static UnmodifiableBloomFilter<String> createFilterFromFile(String fileLocation) {
+        return createFilterFromFile(fileLocation, DEFAULT_TOLERANCE);
+    }
+
+    /**
+     * Creates an instance that will immediately initialize from file.
+     * @param fileLocation The file to load from
+     * @param tolerance The expected false positive tolerance (0.xx-1)
+     * @return A string instance that cannot be modified externally
+     */
+    public static UnmodifiableBloomFilter<String> createFilterFromFile(String fileLocation, double tolerance) {
+        UnmodifiableBloomFilter<String> bloom = createLazyFilterFromFile(fileLocation, tolerance);
+        bloom.ensureInitialized();
+        return bloom;
+    }
+
+    /**
+     * @see UnmodifiableBloomFilter#createLazyFilterFromFile(String, double)
+     */
+    public static UnmodifiableBloomFilter<String> createLazyFilterFromFile(String fileLocation) {
+        return createLazyFilterFromFile(fileLocation, DEFAULT_TOLERANCE);
+    }
+
+    /**
+     * Creates a bloom filter instance that will initialize from file on first use.
+     * @param fileLocation The file to load from
+     * @param tolerance The expected false positive tolerance (0.xx-1)
+     * @return A lazy loading string instance that cannot be modified externally
+     */
+    public static UnmodifiableBloomFilter<String> createLazyFilterFromFile(String fileLocation, double tolerance) {
+        // password filter file is unset, return disabled cache instance without warn
+        if (StringUtil.isNullOrEmpty(fileLocation)) {
+            return new UnmodifiableBloomFilter<String>(null, null, tolerance);
+        }
+        return new UnmodifiableBloomFilter<String>(fileLocation, null, tolerance);
+    }
+
+    private static BloomFilter<String> loadFilter(File file, Double tolerance) {
+        try (
+            BufferedReader reader = new BufferedReader(new FileReader(file), DEFAULT_BUFFER_SIZE)) {
+            // determine entry count for accurate filter creation
+            long entryCount = countEntries(new FileReader(file));
+            ZimbraLog.cache.debug("Creating bloom filter for file with %d entries", entryCount);
+            BloomFilter<String> pendingCache = BloomFilter
+                .create(Funnels.stringFunnel(DEFAULT_CHARSET), entryCount, tolerance);
+            // fill the filter as we read
+            String st;
+            while ((st = reader.readLine()) != null) {
+                pendingCache.put(st);
+            }
+            return pendingCache;
+        } catch (IOException e) {
+            ZimbraLog.cache.warnQuietly("Unable to load bloom filter from file.", e);
+            // return an instance with disabled cache since we failed during read/load
+            return null;
+        }
+    }
+
+    private static long countEntries(FileReader fileReader) throws IOException {
+        try (LineNumberReader reader = new LineNumberReader(fileReader, DEFAULT_BUFFER_SIZE)) {
+            reader.skip(Long.MAX_VALUE);
+            return reader.getLineNumber() + 1L;
+        }
+    }
+}

pkg-builder.pl

@@ -256,6 +256,7 @@ ()
    cpy_file( "store-conf/conf/globs2.zimbra",                      "$stage_base_dir/opt/zimbra/conf/globs2.zimbra" );
    cpy_file( "store-conf/conf/spnego_java_options.in",             "$stage_base_dir/opt/zimbra/conf/spnego_java_options.in" );
    cpy_file( "store-conf/conf/contacts/zimbra-contact-fields.xml", "$stage_base_dir/opt/zimbra/conf/zimbra-contact-fields.xml" );
+   cpy_file( "store-conf/conf/common-passwords.txt",               "$stage_base_dir/opt/zimbra/conf/common-passwords.txt" );
 
    return ["store-conf/conf"];
 }

store-conf/README.md

@@ -14,3 +14,10 @@
 
 - `Everything under build/conf after a build should end up under /opt/zimbra/conf`
 ~                    
+
+
+## Common Password List
+
+- `The common password list is sourced from the [SecLists repository].`
+
+[SecLists repository]: https://github.com/danielmiessler/SecLists

store-conf/build.xml

@@ -1,10 +1,20 @@
 <project name="zm-store-conf" default="conf-dist">
-	<import file="../build-common.xml" />
 	<property name="dist.conf.dir" value="build/conf" />
 	<property name="conf.dir" value="conf" />
 	<property name="conf.cont.dir" value="${conf.dir}/contacts" />
 	<property name="conf.msgs.dir" value="${conf.dir}/msgs" />
+	<property name="conf.common.dir" value="${conf.dir}/common" />
+	<target name="common-passwords-exists">
+		<available file="${conf.common.dir}/common-passwords.gz" property="common-passwords.exists" />
+	</target>
+	<target name="common-passwords" depends="common-passwords-exists" if="common-passwords.exists">
+		<delete file="${conf.dir}/common-passwords.txt" />
+		<untar src="${conf.common.dir}/common-passwords.gz" dest="${conf.dir}/common-passwords.txt" compression="gzip">
+			<globmapper from="common-passwords.txt" to="*" />
+		</untar>
+	</target>
 	<target name="conf-dist">
+		<antcall target="common-passwords" />
 		<copy todir="${dist.conf.dir}" overwrite="true">
 			<fileset dir="${conf.dir}" includes="*.*" />
 		</copy>