From 7f52f2c1a67218da1dd295ab24e2e6982a60dd13 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 24 Jan 2025 17:27:44 -0500 Subject: [PATCH 1/9] Use installed Android cacerts for URLSession --- .../URLSession/libcurl/EasyHandle.swift | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index cdf8875fce..fc87fd447b 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -220,6 +220,64 @@ extension _EasyHandle { try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, caInfo).asError() } return + } else { + // When no certificate file has been specified, assemble all the certificate files + // from the Android certificate store and writes them to a single `cacerts.pem` file + // + // See https://android.googlesource.com/platform/frameworks/base/+/8b192b19f264a8829eac2cfaf0b73f6fc188d933%5E%21/#F0 + + // See https://github.com/apple/swift-nio-ssl/blob/d1088ebe0789d9eea231b40741831f37ab654b61/Sources/NIOSSL/AndroidCABundle.swift#L30 + let certsFolders = [ + "/apex/com.android.conscrypt/cacerts", // >= Android14 + "/system/etc/security/cacerts" // < Android14 + ] + + let aggregateCertPath = NSTemporaryDirectory() + "/cacerts-\(UUID().uuidString).pem" + + if FileManager.default.createFile(atPath: aggregateCertPath, contents: nil) == false { + return + } + + guard let fs = FileHandle(forWritingAtPath: aggregateCertPath) else { + return + } + + // write a header + fs.write(""" + ## Bundle of CA Root Certificates + ## Auto-generated on \(Date()) + ## by aggregating certificates from: \(certsFolders) + + """.data(using: .utf8)!) + + // Go through each folder and load each certificate file (ending with ".0"), + // and append them together into a single aggreagate file tha curl can load. + // The .0 files will contain some extra metadata, but libcurl only cares about the + // -----BEGIN CERTIFICATE----- and -----END CERTIFICATE----- sections, + // so we can naïvely concatenate them all and libcurl will understand the bundle. + for certsFolder in certsFolders { + let certsFolderURL = URL(fileURLWithPath: certsFolder) + if (try? certsFolderURL.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) != true { continue } + let certURLs = try! FileManager.default.contentsOfDirectory(at: certsFolderURL, includingPropertiesForKeys: [.isRegularFileKey, .isReadableKey]) + for certURL in certURLs { + // certificate files have names like "53a1b57a.0" + if certURL.pathExtension != "0" { continue } + do { + if try certURL.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile != true { continue } + if try certURL.resourceValues(forKeys: [.isReadableKey]).isReadable != true { continue } + try fs.write(contentsOf: try Data(contentsOf: certURL)) + } catch { + // ignore individual errors and soldier on… + //logger.warning("bootstrapSSLCertificates: error reading certificate file \(certURL.path): \(error)") + continue + } + } + } + + try! fs.close() + + try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, aggregateCertPath).asError() + return } #endif From 35b9955c597188d40955964b2b73a17966c3b914 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 24 Jan 2025 21:01:44 -0500 Subject: [PATCH 2/9] Fix CFURLSession_easy_setopt_ptr --- .../FoundationNetworking/URLSession/libcurl/EasyHandle.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index fc87fd447b..e2e299c673 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -276,7 +276,9 @@ extension _EasyHandle { try! fs.close() - try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, aggregateCertPath).asError() + aggregateCertPath.withCString { pathPtr in + try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, UnsafeMutablePointer(mutating: pathPtr)).asError() + } return } #endif From 52453cbe9c459b021742a9ca2424fbc78fa27c78 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 25 Jan 2025 08:52:22 -0500 Subject: [PATCH 3/9] Use CFURLSessionOptionCAPATH instead of CFURLSessionOptionCAINFO --- .../URLSession/libcurl/EasyHandle.swift | 57 +++---------------- 1 file changed, 7 insertions(+), 50 deletions(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index e2e299c673..5fbb8afbad 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -221,65 +221,22 @@ extension _EasyHandle { } return } else { - // When no certificate file has been specified, assemble all the certificate files - // from the Android certificate store and writes them to a single `cacerts.pem` file - // - // See https://android.googlesource.com/platform/frameworks/base/+/8b192b19f264a8829eac2cfaf0b73f6fc188d933%5E%21/#F0 - - // See https://github.com/apple/swift-nio-ssl/blob/d1088ebe0789d9eea231b40741831f37ab654b61/Sources/NIOSSL/AndroidCABundle.swift#L30 + // When no certificate file has been specified, check the default Android locations + // like at https://github.com/apple/swift-nio-ssl/blob/main/Sources/NIOSSL/AndroidCABundle.swift let certsFolders = [ "/apex/com.android.conscrypt/cacerts", // >= Android14 "/system/etc/security/cacerts" // < Android14 ] - let aggregateCertPath = NSTemporaryDirectory() + "/cacerts-\(UUID().uuidString).pem" - - if FileManager.default.createFile(atPath: aggregateCertPath, contents: nil) == false { - return - } - - guard let fs = FileHandle(forWritingAtPath: aggregateCertPath) else { - return - } - - // write a header - fs.write(""" - ## Bundle of CA Root Certificates - ## Auto-generated on \(Date()) - ## by aggregating certificates from: \(certsFolders) - - """.data(using: .utf8)!) - - // Go through each folder and load each certificate file (ending with ".0"), - // and append them together into a single aggreagate file tha curl can load. - // The .0 files will contain some extra metadata, but libcurl only cares about the - // -----BEGIN CERTIFICATE----- and -----END CERTIFICATE----- sections, - // so we can naïvely concatenate them all and libcurl will understand the bundle. for certsFolder in certsFolders { - let certsFolderURL = URL(fileURLWithPath: certsFolder) - if (try? certsFolderURL.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) != true { continue } - let certURLs = try! FileManager.default.contentsOfDirectory(at: certsFolderURL, includingPropertiesForKeys: [.isRegularFileKey, .isReadableKey]) - for certURL in certURLs { - // certificate files have names like "53a1b57a.0" - if certURL.pathExtension != "0" { continue } - do { - if try certURL.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile != true { continue } - if try certURL.resourceValues(forKeys: [.isReadableKey]).isReadable != true { continue } - try fs.write(contentsOf: try Data(contentsOf: certURL)) - } catch { - // ignore individual errors and soldier on… - //logger.warning("bootstrapSSLCertificates: error reading certificate file \(certURL.path): \(error)") - continue + var isDirectory: ObjCBool = false + if FileManager.default.fileExists(atPath: certsFolder, isDirectory: &isDirectory), isDirectory == true { + certsFolder.withCString { pathPtr in + try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAPATH, UnsafeMutablePointer(mutating: pathPtr)).asError() } + return } } - - try! fs.close() - - aggregateCertPath.withCString { pathPtr in - try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, UnsafeMutablePointer(mutating: pathPtr)).asError() - } - return } #endif From b367d5bef9a5b6fdb1c726ca20b8ded5a05715d0 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 25 Jan 2025 09:37:24 -0500 Subject: [PATCH 4/9] Fix check for isDirectory.boolValue --- .../FoundationNetworking/URLSession/libcurl/EasyHandle.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index 5fbb8afbad..df29559f55 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -230,7 +230,7 @@ extension _EasyHandle { for certsFolder in certsFolders { var isDirectory: ObjCBool = false - if FileManager.default.fileExists(atPath: certsFolder, isDirectory: &isDirectory), isDirectory == true { + if FileManager.default.fileExists(atPath: certsFolder, isDirectory: &isDirectory), isDirectory.boolValue == true { certsFolder.withCString { pathPtr in try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAPATH, UnsafeMutablePointer(mutating: pathPtr)).asError() } From 2179d94d2c36dd263ba8d56dfde16b3b70d067d5 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 25 Jan 2025 17:12:29 -0500 Subject: [PATCH 5/9] Revert "Fix check for isDirectory.boolValue" This reverts commit b367d5bef9a5b6fdb1c726ca20b8ded5a05715d0. --- .../FoundationNetworking/URLSession/libcurl/EasyHandle.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index df29559f55..5fbb8afbad 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -230,7 +230,7 @@ extension _EasyHandle { for certsFolder in certsFolders { var isDirectory: ObjCBool = false - if FileManager.default.fileExists(atPath: certsFolder, isDirectory: &isDirectory), isDirectory.boolValue == true { + if FileManager.default.fileExists(atPath: certsFolder, isDirectory: &isDirectory), isDirectory == true { certsFolder.withCString { pathPtr in try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAPATH, UnsafeMutablePointer(mutating: pathPtr)).asError() } From b6d7191a5e1766c1caa51af197d97efd0ae922d0 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 25 Jan 2025 17:12:40 -0500 Subject: [PATCH 6/9] Revert "Use CFURLSessionOptionCAPATH instead of CFURLSessionOptionCAINFO" This reverts commit 52453cbe9c459b021742a9ca2424fbc78fa27c78. --- .../URLSession/libcurl/EasyHandle.swift | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index 5fbb8afbad..e2e299c673 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -221,22 +221,65 @@ extension _EasyHandle { } return } else { - // When no certificate file has been specified, check the default Android locations - // like at https://github.com/apple/swift-nio-ssl/blob/main/Sources/NIOSSL/AndroidCABundle.swift + // When no certificate file has been specified, assemble all the certificate files + // from the Android certificate store and writes them to a single `cacerts.pem` file + // + // See https://android.googlesource.com/platform/frameworks/base/+/8b192b19f264a8829eac2cfaf0b73f6fc188d933%5E%21/#F0 + + // See https://github.com/apple/swift-nio-ssl/blob/d1088ebe0789d9eea231b40741831f37ab654b61/Sources/NIOSSL/AndroidCABundle.swift#L30 let certsFolders = [ "/apex/com.android.conscrypt/cacerts", // >= Android14 "/system/etc/security/cacerts" // < Android14 ] + let aggregateCertPath = NSTemporaryDirectory() + "/cacerts-\(UUID().uuidString).pem" + + if FileManager.default.createFile(atPath: aggregateCertPath, contents: nil) == false { + return + } + + guard let fs = FileHandle(forWritingAtPath: aggregateCertPath) else { + return + } + + // write a header + fs.write(""" + ## Bundle of CA Root Certificates + ## Auto-generated on \(Date()) + ## by aggregating certificates from: \(certsFolders) + + """.data(using: .utf8)!) + + // Go through each folder and load each certificate file (ending with ".0"), + // and append them together into a single aggreagate file tha curl can load. + // The .0 files will contain some extra metadata, but libcurl only cares about the + // -----BEGIN CERTIFICATE----- and -----END CERTIFICATE----- sections, + // so we can naïvely concatenate them all and libcurl will understand the bundle. for certsFolder in certsFolders { - var isDirectory: ObjCBool = false - if FileManager.default.fileExists(atPath: certsFolder, isDirectory: &isDirectory), isDirectory == true { - certsFolder.withCString { pathPtr in - try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAPATH, UnsafeMutablePointer(mutating: pathPtr)).asError() + let certsFolderURL = URL(fileURLWithPath: certsFolder) + if (try? certsFolderURL.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) != true { continue } + let certURLs = try! FileManager.default.contentsOfDirectory(at: certsFolderURL, includingPropertiesForKeys: [.isRegularFileKey, .isReadableKey]) + for certURL in certURLs { + // certificate files have names like "53a1b57a.0" + if certURL.pathExtension != "0" { continue } + do { + if try certURL.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile != true { continue } + if try certURL.resourceValues(forKeys: [.isReadableKey]).isReadable != true { continue } + try fs.write(contentsOf: try Data(contentsOf: certURL)) + } catch { + // ignore individual errors and soldier on… + //logger.warning("bootstrapSSLCertificates: error reading certificate file \(certURL.path): \(error)") + continue } - return } } + + try! fs.close() + + aggregateCertPath.withCString { pathPtr in + try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, UnsafeMutablePointer(mutating: pathPtr)).asError() + } + return } #endif From 9fdcfd1d78eb0c11443c8884f5f72e1168d268c5 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 25 Jan 2025 17:20:45 -0500 Subject: [PATCH 7/9] Cleanup and add comments --- .../URLSession/libcurl/EasyHandle.swift | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index e2e299c673..bdc12c6a40 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -224,9 +224,7 @@ extension _EasyHandle { // When no certificate file has been specified, assemble all the certificate files // from the Android certificate store and writes them to a single `cacerts.pem` file // - // See https://android.googlesource.com/platform/frameworks/base/+/8b192b19f264a8829eac2cfaf0b73f6fc188d933%5E%21/#F0 - - // See https://github.com/apple/swift-nio-ssl/blob/d1088ebe0789d9eea231b40741831f37ab654b61/Sources/NIOSSL/AndroidCABundle.swift#L30 + // See https://github.com/apple/swift-nio-ssl/blob/main/Sources/NIOSSL/AndroidCABundle.swift let certsFolders = [ "/apex/com.android.conscrypt/cacerts", // >= Android14 "/system/etc/security/cacerts" // < Android14 @@ -251,7 +249,7 @@ extension _EasyHandle { """.data(using: .utf8)!) // Go through each folder and load each certificate file (ending with ".0"), - // and append them together into a single aggreagate file tha curl can load. + // and append them together into a single aggreagate file that curl can load. // The .0 files will contain some extra metadata, but libcurl only cares about the // -----BEGIN CERTIFICATE----- and -----END CERTIFICATE----- sections, // so we can naïvely concatenate them all and libcurl will understand the bundle. @@ -268,7 +266,6 @@ extension _EasyHandle { try fs.write(contentsOf: try Data(contentsOf: certURL)) } catch { // ignore individual errors and soldier on… - //logger.warning("bootstrapSSLCertificates: error reading certificate file \(certURL.path): \(error)") continue } } @@ -277,6 +274,7 @@ extension _EasyHandle { try! fs.close() aggregateCertPath.withCString { pathPtr in + // note that it would be nice to use CFURLSessionOptionCAPATH instead (https://curl.se/libcurl/c/CURLOPT_CAPATH.html), but it requires a special command to hash the directory contents, which we cannot try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, UnsafeMutablePointer(mutating: pathPtr)).asError() } return From a8bc4f324fb66b5dec1e9c5f8b35b5a6b785a903 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 25 Jan 2025 17:25:41 -0500 Subject: [PATCH 8/9] Update comments --- .../URLSession/libcurl/EasyHandle.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index bdc12c6a40..b9b50a2361 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -223,7 +223,7 @@ extension _EasyHandle { } else { // When no certificate file has been specified, assemble all the certificate files // from the Android certificate store and writes them to a single `cacerts.pem` file - // + // See https://github.com/apple/swift-nio-ssl/blob/main/Sources/NIOSSL/AndroidCABundle.swift let certsFolders = [ "/apex/com.android.conscrypt/cacerts", // >= Android14 @@ -274,7 +274,9 @@ extension _EasyHandle { try! fs.close() aggregateCertPath.withCString { pathPtr in - // note that it would be nice to use CFURLSessionOptionCAPATH instead (https://curl.se/libcurl/c/CURLOPT_CAPATH.html), but it requires a special command to hash the directory contents, which we cannot + // note that it would be nice to use CFURLSessionOptionCAPATH instead + // (see https://curl.se/libcurl/c/CURLOPT_CAPATH.html) + // but it requires `c_rehash` to be run on the folder, which Android doesn't do try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionCAINFO, UnsafeMutablePointer(mutating: pathPtr)).asError() } return From bda9bd480ba534b676d6ca55a465e641de3bc82f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 3 Feb 2025 16:46:24 -0500 Subject: [PATCH 9/9] Just try? to load the certificate Data and continue on failure --- .../FoundationNetworking/URLSession/libcurl/EasyHandle.swift | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift index b9b50a2361..2a22e75b82 100644 --- a/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift +++ b/Sources/FoundationNetworking/URLSession/libcurl/EasyHandle.swift @@ -261,9 +261,7 @@ extension _EasyHandle { // certificate files have names like "53a1b57a.0" if certURL.pathExtension != "0" { continue } do { - if try certURL.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile != true { continue } - if try certURL.resourceValues(forKeys: [.isReadableKey]).isReadable != true { continue } - try fs.write(contentsOf: try Data(contentsOf: certURL)) + try? fs.write(contentsOf: Data(contentsOf: certURL)) } catch { // ignore individual errors and soldier on… continue