import hashlib class BloomFilter: def __init__(self, size=2048, num_hashes=3): self.size = size self.num_hashes = num_hashes self.bits = bytearray(size // 8 + 1) def _hash_positions(self, item): h = hashlib.sha256(item.encode("utf-8")).digest() for i in range(self.num_hashes): val = int.from_bytes(h[i*4:(i+1)*4], "big") % self.size yield val def add(self, item): for pos in self._hash_positions(item): self.bits[pos // 8] |= 1 << (pos % 8) def might_contain(self, item): return all( bool(self.bits[pos // 8] & (1 << (pos % 8))) for pos in self._hash_positions(item) ) @property def bytes(self): return bytes(self.bits) @classmethod def from_bytes(cls, data, size=2048, num_hashes=3): bf = cls(size=size, num_hashes=num_hashes) bf.bits = bytearray(data) return bf @staticmethod def from_items(items, size=2048, num_hashes=3): bf = BloomFilter(size=size, num_hashes=num_hashes) for item in items: bf.add(item) return bf @staticmethod def from_tags(tags, size=2048, num_hashes=3): bf = BloomFilter(size=size, num_hashes=num_hashes) for tag in tags: bf.add(tag.strip().lower()) return bf