Uh oh!
There was an error while loading. Please reload this page.
Uh oh!
There was an error while loading. Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork 51k
Bloom Filter#8615
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Uh oh!
There was an error while loading. Please reload this page.
Bloom Filter #8615
Changes from all commits
173ab0e08bc9700448109486dcbc4111807e6ce098e4d39db76296863926167280ffa05d460aacc5409578d19fd8b1bec028e66912fd7196314237d9b01472c132d50313c80c18e0dde54041ff483a2a0174ce0800cc60e35fa5f55cd20ea7617143799171a1a71f4c4e0263fe746746File filter
Filter by extension
Conversations
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| """ | ||
| See https://en.wikipedia.org/wiki/Bloom_filter | ||
| The use of this data structure is to test membership in a set. | ||
| Compared to Python's built-in set() it is more space-efficient. | ||
| In the following example, only 8 bits of memory will be used: | ||
| >>> bloom = Bloom(size=8) | ||
| Initially, the filter contains all zeros: | ||
| >>> bloom.bitstring | ||
| '00000000' | ||
| When an element is added, two bits are set to 1 | ||
| since there are 2 hash functions in this implementation: | ||
| >>> "Titanic" in bloom | ||
| False | ||
| >>> bloom.add("Titanic") | ||
| >>> bloom.bitstring | ||
| '01100000' | ||
| >>> "Titanic" in bloom | ||
| True | ||
| However, sometimes only one bit is added | ||
| because both hash functions return the same value | ||
| >>> bloom.add("Avatar") | ||
| >>> "Avatar" in bloom | ||
| True | ||
| >>> bloom.format_hash("Avatar") | ||
| '00000100' | ||
| >>> bloom.bitstring | ||
| '01100100' | ||
| Not added elements should return False ... | ||
| >>> not_present_films = ("The Godfather", "Interstellar", "Parasite", "Pulp Fiction") | ||
| >>> { | ||
| ... film: bloom.format_hash(film) for film in not_present_films | ||
| ... } # doctest: +NORMALIZE_WHITESPACE | ||
| {'The Godfather': '00000101', | ||
| 'Interstellar': '00000011', | ||
| 'Parasite': '00010010', | ||
| 'Pulp Fiction': '10000100'} | ||
cclauss marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| >>> any(film in bloom for film in not_present_films) | ||
| False | ||
| but sometimes there are false positives: | ||
| >>> "Ratatouille" in bloom | ||
| True | ||
| >>> bloom.format_hash("Ratatouille") | ||
| '01100000' | ||
| The probability increases with the number of elements added. | ||
| The probability decreases with the number of bits in the bitarray. | ||
| >>> bloom.estimated_error_rate | ||
| 0.140625 | ||
| >>> bloom.add("The Godfather") | ||
| >>> bloom.estimated_error_rate | ||
| 0.25 | ||
| >>> bloom.bitstring | ||
| '01100101' | ||
| """ | ||
| from hashlib import md5, sha256 | ||
| HASH_FUNCTIONS = (sha256, md5) | ||
| class Bloom: | ||
| def __init__(self, size: int = 8) -> None: | ||
| self.bitarray = 0b0 | ||
| self.size = size | ||
| def add(self, value: str) -> None: | ||
| h = self.hash_(value) | ||
| self.bitarray |= h | ||
| def exists(self, value: str) -> bool: | ||
| h = self.hash_(value) | ||
| return (h & self.bitarray) == h | ||
| def __contains__(self, other: str) -> bool: | ||
| return self.exists(other) | ||
| def format_bin(self, bitarray: int) -> str: | ||
| res = bin(bitarray)[2:] | ||
| return res.zfill(self.size) | ||
| @property | ||
| def bitstring(self) -> str: | ||
| return self.format_bin(self.bitarray) | ||
| def hash_(self, value: str) -> int: | ||
cclauss marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| res = 0b0 | ||
| for func in HASH_FUNCTIONS: | ||
| position = ( | ||
| int.from_bytes(func(value.encode()).digest(), "little") % self.size | ||
| ) | ||
| res |= 2**position | ||
| return res | ||
| def format_hash(self, value: str) -> str: | ||
| return self.format_bin(self.hash_(value)) | ||
| @property | ||
| def estimated_error_rate(self) -> float: | ||
cclauss marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| n_ones = bin(self.bitarray).count("1") | ||
| return (n_ones / self.size) ** len(HASH_FUNCTIONS) | ||
Uh oh!
There was an error while loading. Please reload this page.