From a12d2eb17de52cf9015ad09176d1babc2cd8200c Mon Sep 17 00:00:00 2001 From: Justin Mazzocchi <2831158+jzzocc@users.noreply.github.com> Date: Sat, 27 Feb 2021 17:47:00 -0800 Subject: [PATCH] Extract CodableBloomFilter package --- CodableBloomFilter/.gitignore | 5 - CodableBloomFilter/Package.swift | 21 --- .../Sources/CodableBloomFilter/BitArray.swift | 56 -------- .../CodableBloomFilter/BloomFilter.swift | 44 ------ .../DeterministicallyHashable.swift | 7 - .../Data+DeterministicallyHashable.swift | 7 - .../String+DeterministicallyHashable.swift | 7 - .../Sources/CodableBloomFilter/Hash.swift | 49 ------- .../CodableBloomFilterTests.swift | 132 ------------------ Metatext.xcodeproj/project.pbxproj | 2 - .../xcshareddata/swiftpm/Package.resolved | 9 ++ ServiceLayer/Package.swift | 4 +- 12 files changed, 12 insertions(+), 331 deletions(-) delete mode 100644 CodableBloomFilter/.gitignore delete mode 100644 CodableBloomFilter/Package.swift delete mode 100644 CodableBloomFilter/Sources/CodableBloomFilter/BitArray.swift delete mode 100644 CodableBloomFilter/Sources/CodableBloomFilter/BloomFilter.swift delete mode 100644 CodableBloomFilter/Sources/CodableBloomFilter/DeterministicallyHashable.swift delete mode 100644 CodableBloomFilter/Sources/CodableBloomFilter/Extensions/Data+DeterministicallyHashable.swift delete mode 100644 CodableBloomFilter/Sources/CodableBloomFilter/Extensions/String+DeterministicallyHashable.swift delete mode 100644 CodableBloomFilter/Sources/CodableBloomFilter/Hash.swift delete mode 100644 CodableBloomFilter/Tests/CodableBloomFilterTests/CodableBloomFilterTests.swift diff --git a/CodableBloomFilter/.gitignore b/CodableBloomFilter/.gitignore deleted file mode 100644 index 95c4320..0000000 --- a/CodableBloomFilter/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.DS_Store -/.build -/Packages -/*.xcodeproj -xcuserdata/ diff --git a/CodableBloomFilter/Package.swift b/CodableBloomFilter/Package.swift deleted file mode 100644 index 50c8023..0000000 --- a/CodableBloomFilter/Package.swift +++ /dev/null @@ -1,21 +0,0 @@ -// swift-tools-version:5.3 - -import PackageDescription - -let package = Package( - name: "CodableBloomFilter", - products: [ - .library( - name: "CodableBloomFilter", - targets: ["CodableBloomFilter"]) - ], - dependencies: [], - targets: [ - .target( - name: "CodableBloomFilter", - dependencies: []), - .testTarget( - name: "CodableBloomFilterTests", - dependencies: ["CodableBloomFilter"]) - ] -) diff --git a/CodableBloomFilter/Sources/CodableBloomFilter/BitArray.swift b/CodableBloomFilter/Sources/CodableBloomFilter/BitArray.swift deleted file mode 100644 index cef3c88..0000000 --- a/CodableBloomFilter/Sources/CodableBloomFilter/BitArray.swift +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -import Foundation - -struct BitArray { - private var bytes: [UInt8] - - init(data: Data) { - bytes = Array(data) - } -} - -extension BitArray { - var bitCount: Int { bytes.count * UInt8.bitWidth } - - var data: Data { Data(bytes) } - - subscript(index: Int) -> Bool { - get { - let (byteIndex, mask) = Self.byteIndexAndMask(index: index) - - return bytes[byteIndex] & mask > 0 - } - - set { - let (byteIndex, mask) = Self.byteIndexAndMask(index: index) - - if newValue { - bytes[byteIndex] |= mask - } else { - bytes[byteIndex] &= ~mask - } - } - } -} - -extension BitArray: Codable { - init(from decoder: Decoder) throws { - bytes = Array(try decoder.singleValueContainer().decode(Data.self)) - } - - func encode(to encoder: Encoder) throws { - var container = encoder.singleValueContainer() - - try container.encode(Data(bytes)) - } -} - -private extension BitArray { - static func byteIndexAndMask(index: Int) -> (Int, UInt8) { - let (byteIndex, bitIndex) = index.quotientAndRemainder(dividingBy: UInt8.bitWidth) - let mask = UInt8(2 << (bitIndex - 1)) - - return (byteIndex, mask) - } -} diff --git a/CodableBloomFilter/Sources/CodableBloomFilter/BloomFilter.swift b/CodableBloomFilter/Sources/CodableBloomFilter/BloomFilter.swift deleted file mode 100644 index 9313f59..0000000 --- a/CodableBloomFilter/Sources/CodableBloomFilter/BloomFilter.swift +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -import Foundation - -public struct BloomFilter: Codable { - enum CodingKeys: String, CodingKey { - case hashes - case bits = "data" - } - - public let hashes: [Hash] - - private var bits: BitArray - - public init(hashes: Set, byteCount: Int) { - self.init(hashes: hashes, data: Data(repeating: 0, count: byteCount)) - } - - public init(hashes: Set, data: Data) { - // Sort the hashes for consistent decoding output - self.hashes = Array(hashes.sorted { $0.rawValue < $1.rawValue }) - bits = BitArray(data: data) - } -} - -public extension BloomFilter { - var data: Data { bits.data } - - mutating func insert(_ newMember: T) { - for index in indices(newMember) { - bits[index] = true - } - } - - func contains(_ member: T) -> Bool { - indices(member).allSatisfy { bits[$0] } - } -} - -private extension BloomFilter { - func indices(_ member: T) -> [Int] { - hashes.map { abs($0.apply(member)) % bits.bitCount } - } -} diff --git a/CodableBloomFilter/Sources/CodableBloomFilter/DeterministicallyHashable.swift b/CodableBloomFilter/Sources/CodableBloomFilter/DeterministicallyHashable.swift deleted file mode 100644 index 7c9f2b2..0000000 --- a/CodableBloomFilter/Sources/CodableBloomFilter/DeterministicallyHashable.swift +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -import Foundation - -public protocol DeterministicallyHashable { - var dataForHashingDeterministically: Data { get } -} diff --git a/CodableBloomFilter/Sources/CodableBloomFilter/Extensions/Data+DeterministicallyHashable.swift b/CodableBloomFilter/Sources/CodableBloomFilter/Extensions/Data+DeterministicallyHashable.swift deleted file mode 100644 index d336139..0000000 --- a/CodableBloomFilter/Sources/CodableBloomFilter/Extensions/Data+DeterministicallyHashable.swift +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -import Foundation - -extension Data: DeterministicallyHashable { - public var dataForHashingDeterministically: Data { self } -} diff --git a/CodableBloomFilter/Sources/CodableBloomFilter/Extensions/String+DeterministicallyHashable.swift b/CodableBloomFilter/Sources/CodableBloomFilter/Extensions/String+DeterministicallyHashable.swift deleted file mode 100644 index 177f563..0000000 --- a/CodableBloomFilter/Sources/CodableBloomFilter/Extensions/String+DeterministicallyHashable.swift +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -import Foundation - -extension String: DeterministicallyHashable { - public var dataForHashingDeterministically: Data { Data(utf8) } -} diff --git a/CodableBloomFilter/Sources/CodableBloomFilter/Hash.swift b/CodableBloomFilter/Sources/CodableBloomFilter/Hash.swift deleted file mode 100644 index 90f0f97..0000000 --- a/CodableBloomFilter/Sources/CodableBloomFilter/Hash.swift +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -import Foundation - -public enum Hash: String, Codable { - case djb232 - case djb2a32 - case sdbm32 - case fnv132 - case fnv1a32 -} - -extension Hash { - func apply(_ hashable: DeterministicallyHashable) -> Int { - Int(Array(hashable.dataForHashingDeterministically) - .map(UInt32.init) - .reduce(offsetBasis, hash)) - } -} - -// http://www.cse.yorku.ca/~oz/hash.html -// http://www.isthe.com/chongo/tech/comp/fnv/ - -private extension Hash { - static let fnvPrime: UInt32 = 16777619 - - var offsetBasis: UInt32 { - switch self { - case .djb232, .djb2a32: return 5381 - case .sdbm32: return 0 - case .fnv132, .fnv1a32: return 2166136261 - } - } - - func hash(result: UInt32, next: UInt32) -> UInt32 { - switch self { - case .djb232: - return (result << 5) &+ result &+ next - case .djb2a32: - return (result << 5) &+ result ^ next - case .sdbm32: - return next &+ (result << 6) &+ (result << 16) &- result - case .fnv132: - return (result &* Self.fnvPrime) ^ next - case .fnv1a32: - return (result ^ next) &* Self.fnvPrime - } - } -} diff --git a/CodableBloomFilter/Tests/CodableBloomFilterTests/CodableBloomFilterTests.swift b/CodableBloomFilter/Tests/CodableBloomFilterTests/CodableBloomFilterTests.swift deleted file mode 100644 index 2052140..0000000 --- a/CodableBloomFilter/Tests/CodableBloomFilterTests/CodableBloomFilterTests.swift +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright © 2020 Metabolist. All rights reserved. - -@testable import CodableBloomFilter -import XCTest - -final class CodableBloomFilterTests: XCTestCase { - func testHashes() { - XCTAssertEqual(Hash.djb232.apply("hash"), 2090320585) - XCTAssertEqual(Hash.djb2a32.apply("hash"), 2087809207) - XCTAssertEqual(Hash.sdbm32.apply("hash"), 385600046) - XCTAssertEqual(Hash.fnv132.apply("hash"), 3616638997) - XCTAssertEqual(Hash.fnv1a32.apply("hash"), 3469047761) - } - - func testContains() throws { - var sut = BloomFilter(hashes: [.sdbm32, .djb232], byteCount: 8) - - sut.insert("lol") - sut.insert("ok") - - XCTAssert(sut.contains("lol")) - XCTAssert(sut.contains("ok")) - XCTAssertFalse(sut.contains("wtf")) - XCTAssertFalse(sut.contains("no")) - } - - func testData() throws { - var sut = BloomFilter(hashes: [.sdbm32, .djb232], byteCount: 8) - - sut.insert("lol") - sut.insert("ok") - - XCTAssertEqual(sut.data, Data([0, 16, 0, 0, 0, 2, 0, 144])) - } - - func testFromData() throws { - let sut = BloomFilter(hashes: [.sdbm32, .djb232], data: Data([0, 16, 0, 0, 0, 2, 0, 144])) - - XCTAssert(sut.contains("lol")) - XCTAssert(sut.contains("ok")) - XCTAssertFalse(sut.contains("wtf")) - XCTAssertFalse(sut.contains("no")) - } - - func testCoding() throws { - var sut = BloomFilter(hashes: [.sdbm32, .djb232], byteCount: 8) - let expectedData = Data(#"{"data":"ABAAAAACAJA=","hashes":["djb232","sdbm32"]}"#.utf8) - - sut.insert("lol") - sut.insert("ok") - - let encoder = JSONEncoder() - - encoder.outputFormatting = .sortedKeys - - let data = try encoder.encode(sut) - - XCTAssertEqual(data, expectedData) - - let decoded = try JSONDecoder().decode(BloomFilter.self, from: data) - - XCTAssert(decoded.contains("lol")) - XCTAssert(decoded.contains("ok")) - XCTAssertFalse(decoded.contains("wtf")) - XCTAssertFalse(decoded.contains("no")) - } - - func testInvalidHash() throws { - let invalidData = Data(#"{"data":"ABAAAAACAJA=","hashes":["djb232","invalid"]}"#.utf8) - - XCTAssertThrowsError(try JSONDecoder().decode(BloomFilter.self, from: invalidData)) { - guard case DecodingError.dataCorrupted = $0 else { - XCTFail("Expected data corrupted error") - - return - } - } - } - - func testDataEncodingStrategy() throws { - var sut = BloomFilter(hashes: [.sdbm32, .djb232], byteCount: 8) - let expectedData = Data(#"{"data":"0010000000020090","hashes":["djb232","sdbm32"]}"#.utf8) - - sut.insert("lol") - sut.insert("ok") - - let encoder = JSONEncoder() - - encoder.outputFormatting = .sortedKeys - encoder.dataEncodingStrategy = .custom { data, encoder in - var container = encoder.singleValueContainer() - - try container.encode(data.map { String(format: "%02.2hhx", $0) }.joined()) - } - - let data = try encoder.encode(sut) - - XCTAssertEqual(data, expectedData) - } - - func testDataDecodingStrategy() throws { - let data = Data(#"{"data":"0010000000020090","hashes":["djb232","sdbm32"]}"#.utf8) - let decoder = JSONDecoder() - - decoder.dataDecodingStrategy = .custom { decoder in - let container = try decoder.singleValueContainer() - let string = try container.decode(String.self) - var bytes = [UInt8]() - var i = string.startIndex - - while i != string.endIndex { - let j = string.index(i, offsetBy: 2) - - guard let byte = UInt8(string[i...self, from: data) - - XCTAssert(sut.contains("lol")) - XCTAssert(sut.contains("ok")) - XCTAssertFalse(sut.contains("wtf")) - XCTAssertFalse(sut.contains("no")) - } -} diff --git a/Metatext.xcodeproj/project.pbxproj b/Metatext.xcodeproj/project.pbxproj index 6579e9d..eb26031 100644 --- a/Metatext.xcodeproj/project.pbxproj +++ b/Metatext.xcodeproj/project.pbxproj @@ -396,7 +396,6 @@ D0D2AC4C25BCD2A9003D5DF2 /* TagTableViewCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TagTableViewCell.swift; sourceTree = ""; }; D0D2AC5225BCD2BA003D5DF2 /* TagContentConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TagContentConfiguration.swift; sourceTree = ""; }; D0D2AC6625BD0484003D5DF2 /* LineChartView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LineChartView.swift; sourceTree = ""; }; - D0D7C013250440610039AD6F /* CodableBloomFilter */ = {isa = PBXFileReference; lastKnownFileType = folder; path = CodableBloomFilter; sourceTree = ""; }; D0D93EB925D9C70400C622ED /* AutocompleteItemView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutocompleteItemView.swift; sourceTree = ""; }; D0D93EBF25D9C71D00C622ED /* AutocompleteItemContentConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutocompleteItemContentConfiguration.swift; sourceTree = ""; }; D0D93ECF25D9C9ED00C622ED /* AutocompleteItemCollectionViewCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutocompleteItemCollectionViewCell.swift; sourceTree = ""; }; @@ -635,7 +634,6 @@ D0477F2A25C6EB90005C5368 /* Activities */, D0C7D45224F76169001EBDBB /* Assets.xcassets */, D0FE1C9625368A15003EF1EB /* Caches */, - D0D7C013250440610039AD6F /* CodableBloomFilter */, D0A1F4F5252E7D2A004435BF /* Data Sources */, D085C3BB25008DEC008A6C5E /* DB */, D0C7D46824F76169001EBDBB /* Extensions */, diff --git a/Metatext.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Metatext.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 36686dc..da39d70 100644 --- a/Metatext.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Metatext.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -10,6 +10,15 @@ "version": "1.0.0" } }, + { + "package": "CodableBloomFilter", + "repositoryURL": "https://github.com/metabolist/codable-bloom-filter.git", + "state": { + "branch": null, + "revision": "53f80465056a95253df797f1d8a9199f8b1c0543", + "version": "1.0.0" + } + }, { "package": "CombineExpectations", "repositoryURL": "https://github.com/groue/CombineExpectations.git", diff --git a/ServiceLayer/Package.swift b/ServiceLayer/Package.swift index f28809b..fe722b9 100644 --- a/ServiceLayer/Package.swift +++ b/ServiceLayer/Package.swift @@ -18,7 +18,9 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/groue/CombineExpectations.git", .upToNextMajor(from: "0.7.0")), - .package(path: "CodableBloomFilter"), + .package(name: "CodableBloomFilter", + url: "https://github.com/metabolist/codable-bloom-filter.git", + .upToNextMajor(from: "1.0.0")), .package(path: "DB"), .package(path: "Keychain"), .package(path: "MastodonAPI"),