# -*- encoding: utf-8 -*-
"""
KERI
keri.db.subing module
Provide variety of mixin classes for LMDB sub-dbs with various behaviors.
Takes of advantage of multiple inheritance to enable mixtures of behaviors
with minimal code duplication (more DRY).
New style python classes use the C3 linearization algorithm. Multiple inheritance
forms a directed acyclic graph called a diamond graph. This graph is linarized
into the method resolution order.
Use class.mro() or class.__mro__
(see https://www.geeksforgeeks.org/method-resolution-order-in-python-inheritance/)
Basically:
* children always precede their parents
* immediate parent classes of a child are visited in the order listed in the
child class statement.
* a super class is visited only after all sub classes have been visited
* linearized graph is monotonic (a class is only visted once)
Principally:
SuberBase class provides trim, cnt, getTopItemIter, and getFullItemIter
Suber subclass of SuberBase also provides, put, pin, get, and rem methods
Suber is the simple class for managing a serialized value in a subdb with a
set of keys as a tuple (iterable) that is converted to a .sep delimited key
that defines the key space.
CesrSuber class extends Suber for values that are serializations of CESR serializable
object instances. Ducktyped subclasses of Matter, Indexer, and Counter or the like.
IoSetSuber class extends Suber to allow a set of values to be stored in insertion
order at each effective key. Only one copy of a unique value is allowed in the
set at a given effective key. The effective key suffixes an ordinal to the key
space to track insertion ordering. IoSetSuber adds additional methods to manage
IoSets of values.
CatCesrSuber adds the ability to store multiple concatenated serializations at
a value
CatCesrIoSetSuber combines the capabilities
Other special classer for special values
SerderSuber stores Serialized Serder Instances of in JSON, CBOR, or MGPK
Also for Secrets private keys
SignerSuber
CryptSignerSuber
Also for using the dupsort==true mechanism is
DupSuber
CesrDupSuber
Class Architecture
Suber is simple lexographic database with only one value per key
OnSuber is simple lexographic database where trailing part of key is serialized
ordinal number so that the ordering within each key prefix is monotonically
increasing numeric
B64Suber provides separated fields of B64 primitives for values. Useful when don't
need to CESR ser/des the primitives or performance
The term 'set' of values means that no value may appear more than once in the set.
Sets support idempotent adds and puts to db. This means one can add or put the same
(key, val) pair multiple times and not change the db.
DupSuber provides set of lexicographic ordered values at each key. Each value has
a limited size (key + value <= 511 byes). The set is performant. Good for indices.
IoDupSuber provides set of insertion ordered values at each key. Each value has
a limited size (key + value <= 511 byes). The set is less perfromant than DupSuber
but more performant than IoSetSuber. Good for insertion ordered indices
IoSetSuber proves set of insertion ordered values at each key. Value size is not limited
Good for any insertion ordered set where size may be too large for IoDupSuber
OnIoDupSuber provides set of insertion ordered values where the where trailing
part of key is serialized ordinal number so that the ordering within each
key prefix is monotonically increasing numeric. Useful to provide omndices
for sn ordering of superseding KEL events.
Each of these base types for managing the key space may be mixed with other
Classes that provide different types of values these include.
Cesr
CatCesr
Serder
etc.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Type, Union
from collections.abc import Iterable
from hio.help import ogler
from ..help import helping
from .dbing import LMDBer
if TYPE_CHECKING:
from ..core import coring, scheming, serdering, signing
logger = ogler.getLogger()
[docs]
class SuberBase():
"""
Base class for Sub DBs of LMDBer
Provides common methods for subclasses
Do not instantiate but use a subclass
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
Sep = '.' # separator for combining key iterables
[docs]
def __init__(self, db: LMDBer, *,
subkey: str='docs.',
dupsort: bool=False,
sep: str=None,
verify: bool=False,
**kwa):
"""
Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(SuberBase, self).__init__() # for multi inheritance
self.db = db
self.sdb = self.db.env.open_db(key=subkey.encode("utf-8"), dupsort=dupsort)
self.sep = sep if sep is not None else self.Sep
self.verify = True if verify else False
def _tokey(self, keys: str|bytes|memoryview|Iterable, topive: bool=False):
"""
Converts keys to key bytes with proper separators and returns key bytes.
If keys is already str or bytes or memoryview then returns key bytes.
Else If keys is iterable (non-str) of strs or bytes then joins with
separator converts to key bytes and returns. When keys is iterable and
topive is True then enables partial key from top branch of key space given
by partial keys by appending separator to end of partial key
Returns:
key (bytes): each element of keys is joined by .sep. If topive then
last char of key is .sep
Parameters:
keys (str|bytes|memoryview|Iterable[str|bytes|memoryview]): db key or
Iterable of (str|bytes|memoryview) to form key.
topive (bool): True means treat as partial key tuple from top branch of
key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of topive value
"""
if hasattr(keys, "encode"): # str
return keys.encode()
if isinstance(keys, memoryview): # memoryview of bytes
return bytes(keys) # return bytes
elif hasattr(keys, "decode"): # bytes
return keys
if topive and keys[-1]: # topive and keys is not already partial
keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep
return (self.sep.join(key if hasattr(key, "encode") else bytes(key).decode()
for key in keys).encode()) # bytes(key) converts memoryview
def _tokeys(self, key: str|bytes|memoryview):
"""
Converts key bytes to keys tuple of strs by decoding and then splitting
at separator .sep.
Returns:
keys (tuple[str]): makes tuple by splitting key at sep
Parameters:
key (str|bytes|memoryview): db key.
"""
if isinstance(key, memoryview): # memoryview of bytes
key = bytes(key)
if hasattr(key, "decode"): # bytes
key = key.decode() # convert to str
return tuple(key.split(self.sep))
def _ser(self, val: str | bytes | memoryview):
"""
Serialize value to bytes to store in db
Parameters:
val (str | bytes | memoryview): encodable as bytes
"""
if isinstance(val, memoryview): # memoryview is always bytes
val = bytes(val) # return bytes
return (val.encode() if hasattr(val, "encode") else val)
def _des(self, val: bytes | memoryview):
"""
Deserialize val to str
Parameters:
val (bytes | memoryview): decodable as str
"""
if isinstance(val, memoryview): # memoryview is always bytes
val = bytes(val) # convert to bytes
return (val.decode() if hasattr(val, "decode") else val)
[docs]
def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False):
"""Removes all entries in top branch of db given by keys.
Enables removal of whole branches of db key space.
Returns:
result (bool): True if val at key exists so delete successful.
False otherwise
Parameters:
keys (str|bytes|memoryview|Iterable): of key parts that may be
a truncation of a full keys tuple in in order to address all the
items from multiple branches of the key space.
If keys is empty then trims all items in database.
Either append "" to end of keys Iterable to ensure get properly
separated top branch key or use topive=True.
topive (bool): True means treat as partial key tuple from top branch
of key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of top value
Uses python .startswith() to match keyspace since str.startswith('')
always returns True so empty str will match all keys in db.
"""
return self.db.remTop(db=self.sdb, top=self._tokey(keys, topive=topive))
remTop = trim # alias for convenience
[docs]
def cntTop(self, keys: str|bytes|memoryview|Iterable="",
*, topive=False):
"""Counts all entries in top branch of db given by keys.
When keys is empty then counts all entries in whole db.
Returns:
cnt (int): count of all entries in top branch of sdb
Parameters:
keys (str|bytes|memoryview|Iterable): of key
parts that may be a truncation of a full keys tuple in
in order to address all the items from multiple branches of the
key space.
If keys is empty then gets all items in database.
Either append "" to end of keys Iterable to ensure get properly
separated top branch key or use topive=True.
topive (bool): True means treat keys as delimited top of partial
branch in key space by forcing resultant key to end in .sep
character.
False means treat keys as undelimited partial or full branch in
key space by not forcing resultant key to ebd in .sep character.
When last item in keys is empty str then will treat as delimited
partial branch ending in .sep regardless of topive value.
Uses python .startswith() to match keyspace since str.startswith('')
always returns True so empty str will match all keys in db.
"""
return self.db.cntTop(db=self.sdb, top=self._tokey(keys, topive=topive))
[docs]
def cntAll(self):
"""Counts all the entries in subdb.
Should be overidden in subclasses with parameters to count more
specifically.
Returns:
cnt (int): count of all entries in sdb
"""
return self.db.cntAll(db=self.sdb)
cnt = cntAll # migration alias for backward compt
[docs]
def getTopItemIter(self, keys: str|bytes|memoryview|Iterable="",
*, topive=False):
"""Iterates over all the items in top branch defined by keys where
keys may be a truncation of a full branch. The truncation format may be
modified by the topive parameter.
Iterates over top branch of items in .db subclasses that do special
hidden transforms on either the keyspace or valuespace.
Should override this method to hide hidden parts from the returned items.
For example, adding either a hidden key space suffix or hidden val space
proem to ensure insertion order.
Use getFullItemIter instead to return full items with hidden parts
shown for debugging or testing.
Returns:
items (Iterator[tuple[key,val]]): (key, val) tuples of each item
over the all the items in subdb whose key startswith key made from
keys. Keys may be keyspace prefix to return branches of key space.
When keys is empty then returns all items in subdb
Parameters:
keys (str|bytes|memoryview|Iterable): of key
parts that may be a truncation of a full keys tuple in
in order to address all the items from multiple branches of the
key space.
If keys is empty then gets all items in database.
Either append "" to end of keys Iterable to ensure get properly
separated top branch key or use topive=True.
topive (bool): True means treat keys as delimited top of partial
branch in key space by forcing resultant key to end in .sep
character.
False means treat keys as undelimited partial or full branch in
key space by not forcing resultant key to ebd in .sep character.
When last item in keys is empty str then will treat as delimited
partial branch ending in .sep regardless of topive value.
Uses python .startswith() to match keyspace since str.startswith('')
always returns True so empty str will match all keys in db.
"""
for key, val in self.db.getTopItemIter(db=self.sdb,
top=self._tokey(keys, topive=topive)):
yield (self._tokeys(key), self._des(val))
[docs]
def getFullItemIter(self, keys: str|bytes|memoryview|Iterable="",
*, topive=False):
"""Iterator over items in .db that returns full items with subclass
specific special hidden parts shown for debugging or testing.
Returns:
items (Iterator[tuple[key,val]]): (key, val) tuples of each item
over the all the items in subdb whose key startswith key made from
keys. Keys may be keyspace prefix to return branches of key space.
When keys is empty then returns all items in subdb.
This is meant to return full parts of items in both keyspace and
valuespace which may be useful in debugging or testing.
Parameters:
keys (str|bytes|memoryview|Iteratable): of key parts that may be
a truncation of a full keys tuple in in order to address all the
items from multiple branches of the key space.
If keys is empty then gets all items in database.
Either append "" to end of keys Iterable to ensure get properly
separated top branch key or use topive=True.
topive (bool): True means treat as partial key tuple from top branch of
key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of top value
Uses python .startswith() to match keyspace since str.startswith('')
always returns True so empty str will match all keys in db.
"""
for key, val in self.db.getTopItemIter(db=self.sdb,
top=self._tokey(keys, topive=topive)):
yield (self._tokeys(key), self._des(val))
[docs]
class Suber(SuberBase):
"""
Subclass of SuberBase with no LMDB duplicates (i.e. multiple values at same key).
"""
[docs]
def __init__(self, db: LMDBer, *,
subkey: str = 'docs.',
dupsort: bool=False, **kwa):
"""Initialze instance
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to False
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
"""
super(Suber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa)
[docs]
def put(self, keys: Union[str, Iterable], val: Union[bytes, str, any]):
"""Puts val at key made from keys. Does not overwrite
Parameters:
keys (tuple): of key strs to be combined in order to form key
val (bytes): value
Returns:
result (bool): True If successful, False otherwise, such as key
already in database.
"""
return (self.db.putVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val)))
[docs]
def pin(self, keys: Union[str, Iterable], val: Union[bytes, str]):
"""Pins (sets) val at key made from keys. Overwrites.
Parameters:
keys (tuple): of key strs to be combined in order to form key
val (bytes): value
Returns:
result (bool): True If successful. False otherwise.
"""
return (self.db.setVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val)))
[docs]
def get(self, keys: Union[str, Iterable]):
"""Gets val at keys
Parameters:
keys (tuple): of key strs to be combined in order to form key
Returns:
data (str): decoded as utf-8
None if no entry at keys
Usage:
Use walrus operator to catch and raise missing entry
if (data := mydb.get(keys)) is None:
raise ExceptionHere
use data here
"""
val = self.db.getVal(db=self.sdb, key=self._tokey(keys))
return (self._des(val) if val is not None else None)
[docs]
def rem(self, keys: Union[str, Iterable]):
"""Removes entry at keys. This safe if keys empty or missing then
Parameters:
keys (tuple): of key strs to be combined in order to form key
Returns:
result (bool): True if key exists so delete successful.
False if key empty or missing from db
Raises KeyError if key to big or otherwise bad
"""
return(self.db.remVal(db=self.sdb, key=self._tokey(keys)))
[docs]
def cnt(self):
"""Counts all the entries in subdb.
Should be overidden in subclasses with parameters to count more
specifically.
Returns:
cnt (int): count of all entries in sdb
"""
# for non-collective non-on subers cnt is cntAll
return self.db.cntAll(db=self.sdb)
[docs]
class OnSuberBase(SuberBase):
"""
Subclass of SuberBase that adds methods for keys with exposed trialing
ordinal key part that is 32 byte serializaton of monotonically increasing
ordinal number on such as sn or fn. Useful for escrows that are ordered
by ordinals such as first seen or sequence number.
Each key consistes of top key joined with .sep to ordinal tail
Works with dupsort==True or False
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Default False
sep (str): separator to convert keys iterator to key bytes for db key
Default '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(OnSuberBase, self).__init__(*pa, **kwa)
[docs]
def put(self, keys: str|bytes|memoryview|Iterable, on: int=0,
val: str|bytes|memoryview|None=None):
"""
Returns
result (bool): True if onkey made from key+sep+serialized on is
not found in database so value is written
idempotently.
False otherwise
Parameters:
keys (str|bytes|memoryview|Iterable): keys as prefix to be
combined with serialized exposed on tail and sep to form onkey
on (int): ordinal number used with onKey(key ,on) to form key.
val (str|bytes|memoryview|None): serialized value to put
When None returns False
"""
if val is None:
return False
return (self.db.putOnVal(db=self.sdb,
key=self._tokey(keys),
on=on,
val=self._ser(val),
sep=self.sep.encode()))
[docs]
def pin(self, keys: str|bytes|memoryview|Iterable, on: int=0,
val: str|bytes|memoryview|None=None):
"""
Returns
result (bool): True if value is written or overwritten at onkey
False otherwise
Parameters:
keys (str|bytes|memoryview|Iterable): keys as prefix to be
combined with serialized exposed on tail and sep to form onkey
on (int): ordinal number used with onKey(key ,on) to form key.
val (str|bytes|memoryview|None): serialized value to pin
when None returns False
"""
if val is None:
return False
return (self.db.pinOnVal(db=self.sdb,
key=self._tokey(keys),
on=on,
val=self._ser(val),
sep=self.sep.encode()))
[docs]
def append(self, keys: str|bytes|memoryview|Iterable,
val: str|bytes|memoryview):
"""Appends val to next highest unused exposed ordinal tail and returns the
ordinal.
Returns:
on (int): ordinal number of newly appended val
Parameters:
keys (str|bytes|memoryview|Iterable): top keys as prefix to be
combined with serialized exposed on tail and sep to form key
if key empty then raises ValueError
val (str|bytes|memoryview): serialized value to append
If None then raises ValueError
"""
return (self.db.appendOnVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val),
sep=self.sep.encode()))
[docs]
def getItem(self, keys: str|bytes|memoryview|Iterable, on: int=0):
"""Gets item (key, on, val) at onkey made from keys and on.
When onkey missing or key empty or None returns None
Returns
item (tuple[bytes, int, str|bytes|memoryview]|None): at onkey if any
of form (key, on, val)
None if no entry at onkey or key empty or None
Parameters:
keys (str|bytes|memoryview|Iterable): keys as prefix to be
combined with serialized exposed on tail and sep to form onkey
on (int): ordinal number used with onKey(key ,on) to form key.
"""
if (item := self.db.getOnItem(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())) is None:
return None
key, on, val = item
return (self._tokeys(key), on, self._des(val))
[docs]
def get(self, keys: str|bytes|memoryview|Iterable, on: int=0):
"""Gets val at onkey made from keys and on.
Returns
val (str): serialization at onkey if any
None if no entry at onkey
Parameters:
keys (str|bytes|memoryview|Iterable): keys as prefix to be
combined with serialized exposed on tail and sep to form onkey
on (int): ordinal number used with onKey(key ,on) to form key.
"""
val = self.db.getOnVal(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())
return (self._des(val) if val is not None else None)
[docs]
def rem(self, keys: str|bytes|memoryview|Iterable, on: int=0):
"""Removes entry at onkey = key + sep + on
When key is missing or empty or None returns False
Returns
result (bool): True if onkey made from key+sep+serialized on is
found in database so entry at onkey is removed
False otherwise if no entry at onkey or key is empty.
Parameters:
keys (str|bytes|memoryview|Iterable): keys as prefix to be
combined with serialized exposed on tail and sep to form onkey
on (int): ordinal number used with onKey(key ,on) to form key.
"""
return (self.db.remOn(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode()))
[docs]
def remAll(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Removes entry for each on >= on at key.
When on == 0, default, then removes each entry for all on at key.
When key is empty then removes whole db.
Returns:
result (bool): True if onkey with dup val exists so rem successful.
False otherwise
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
When key is empty then remove all entries in whole db
on (int): base key. None means all on for key
"""
return self.db.remOnAll(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())
[docs]
def cnt(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Counts all entries with same key over all all on >= on.
If key not in db then count is 0
Returns
cnt (int): count of of all exposed on tail keyed vals with same
onkey prefix but different on in onkey in db starting at ordinal
number on where key is formed with onKey(key,on). Count at
each onkey includes duplicates if any.
Parameters:
keys (str|bytes|memoryview|Iterable): top keys as prefix to be
combined with serialized exposed on tail and sep to form top key
When keys is empty then counts whole database including
duplicates if any.
on (int): ordinal number used with onKey(key,on) to form key.
"""
if not keys:
return 0
return (self.db.cntOnAll(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode()))
[docs]
def cntAll(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Counts all entries with same key over all all on >= on. When keys is
empty then counts for on for all keys in whole database.
Returns
cnt (int): count of of all exposed on tail keyed vals with same
onkey prefix but different on in onkey in db starting at ordinal
number on where key is formed with onKey(key,on). Count at
each onkey includes duplicates if any.
Parameters:
keys (str|bytes|memoryview|Iterable): top keys as prefix to be
combined with serialized exposed on tail and sep to form top key
When keys is empty then counts whole database including
duplicates if any.
on (int): ordinal number used with onKey(key,on) to form key.
"""
return (self.db.cntOnAll(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode()))
[docs]
def getTopItemIter(self, keys: str|bytes|memoryview|Iterable=""):
"""Iterates over top branch of all entries where each top key startwith
key made from keys.
Assumes every effective key in db has trailing on element,
onkey = key + sep + on, so can return on in item.
When top key is empty, gets all items in database.
Returns:
items (Iterator[(tuple, int, str)]): iterator of triples
(keys, on, val)
where keys forms base key, on is int, and val is entry value at
with insertion ordering suffix removed from effective key.
Parameters:
keys (str|bytes|memoryview|Iterable): keys as truncated top key,
to get a key space prefix to get all the items
from multiple branches of the key space.
If top key is empty then gets all items in database.
on (int): ordinal number used with onKey(pre,on) to form key.
"""
for key, on, val in (self.db.getOnTopItemIter(db=self.sdb,
top=self._tokey(keys),
sep=self.sep.encode())):
yield (self._tokeys(key), on, self._des(val))
[docs]
def getAllItemIter(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Iterates over all entries in db for all onkey made from keys and on
for all on >= on.
When on==0 default then gets all on.
When keys empty then iterates over whole db.
Returns:
items (Iterator[(key, on, val)]): triples of key, on, val with same
key but increments of on >= on i.e. all onkey beginning with on
Parameters:
keys (str|bytes|memoryview|iterator): keys as prefix to be
combined with serialized on exposed on tail and sep to form actual key
on (int): ordinal number used with onKey(pre,on) to form key at at
which to initiate retrieval.
When on==0 then all on for key
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllIter(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""
Returns:
items (Iterator[bytes]): of val with same key but increments of
on >= on i.e. all key.on beginning with on
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized exposed on tail and sep to form actual key
When keys is empty then retrieves whole database including
duplicates if any
on (int): ordinal number used with onKey(pre,on) to form key at at
which to initiate retrieval
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._des(val))
[docs]
class OnSuber(OnSuberBase, Suber):
"""
Subclass of OnSuberBase andSuber that adds methods for keys with ordinal
numbered exposed tail.
Each key consistes of pre joined with .sep to ordinal tail
Assumes dupsort==False
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to False
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(OnSuber, self).__init__(*pa, **kwa)
[docs]
class B64SuberBase(SuberBase):
"""
Base Class whose values are Iterables of Base64 str or bytes that are stored
in db as .sep joined Base64 bytes. Separator character must not be valid
Base64 character so the split will work unambiguously.
Automatically joins and splits along separator to Iterable (tuple) of Base64
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
for db key and also used to convert val iterator to val bytes
Must not be Base64 character.
default is self.Sep == '.'
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
Must not be Base64 character.
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(B64SuberBase, self).__init__(*pa, **kwa)
if helping.Reb64.match(self.sep.encode()):
raise ValueError("Invalid sep={self.sep}, must not be Base64 char.")
def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]):
"""Converts vals to val bytes with proper separators and returns val bytes.
If vals is already str or bytes or memoryview then returns val bytes.
Else If vals is iterable (non-str) of strs or bytes or memoryview then
joins with .sep and converts to val bytes and returns.
Returns:
val (bytes): each element of vals is joined by .sep.
Parameters:
vals (str | bytes | memoryview | Iterable[str | bytes]): db val or
Iterable of (str | bytes | memoryview) to form val.
Note, join of bytes sep works with memoryview.
"""
if hasattr(vals, "encode"): # str
val = vals.encode("utf-8")
if not (helping.Reb64.match(val)):
raise ValueError(f"Non Base64 {val=}.")
return val
if isinstance(vals, memoryview): # memoryview of bytes
val = bytes(vals) # return bytes
if not (helping.Reb64.match(val)):
raise ValueError(f"Non Base64 {val=}.")
return val
elif hasattr(vals, "decode"): # bytes
val = vals
if not (helping.Reb64.match(val)):
raise ValueError(f"Non Base64 {val=}.")
return val
vals = tuple(v.encode() if hasattr(v, "encode") else v for v in vals) # make bytes
for val in vals:
if not (helping.Reb64.match(val)):
raise ValueError(f"Non Base64 {val=}.")
return (self.sep.encode().join(vals))
def _tovals(self, val: bytes | memoryview):
"""
Converts val bytes to vals tuple of strs by decoding and then splitting
at separator .sep.
Returns:
vals (tuple[str]): makes tuple by splitting val at .sep
Parameters:
val (bytes | memoryview): db Base64 val.
"""
if isinstance(val, memoryview): # memoryview of bytes
val = bytes(val)
if hasattr(val, "decode"): # bytes
val = val.decode("utf-8") # convert to str
return tuple(val.split(self.sep))
def _ser(self, val: Union[Iterable, str, bytes]):
"""
Serialize val to bytes to store in db
When val is Iterable then joins each elements with .sep returns val bytes
Returns:
val (bytes): .sep join of each Base64 bytes in val
Parameters:
val (Union[Iterable, bytes]): of Base64 bytes
"""
if not helping.isNonStringIterable(val): # not iterable
val = (val, ) # make iterable
return (self._toval(val))
def _des(self, val: memoryview | bytes):
"""
Converts val bytes to vals tuple of subclass instances by deserializing
.qb64b concatenation in order of each instance in .klas
Returns:
vals (tuple): subclass instances
Parameters:
val (Union[bytes, memoryview]): of concatenation of .qb64b
"""
return self._tovals(val)
[docs]
class B64Suber(B64SuberBase, Suber):
"""
Subclass of B64SuberBase and Suber that serializes and deserializes values
as .sep joined strings of Base64 components.
.sep must not be Base64 character.
Assumes dupsort==False
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to False
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
also used to convert val iterator to val bytes
Must not be Base64 character.
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(B64Suber, self).__init__(*pa, **kwa)
[docs]
class CesrSuberBase(SuberBase):
"""Sub class of SuberBase where data is CESR encode/decode ducktyped subclass
instance such as Matter, Indexer, Counter with .qb64b property when provided
as fully qualified serialization
Automatically serializes and deserializes from qb64b to/from CESR instance
._ser override .put .set input value to be instance that is serialized
"""
[docs]
def __init__(self,
*pa,
klas: Type[coring.Matter] | None = None,
strict: bool = False,
**kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
Parameters:
klas (Type[coring.Matter]): Class reference to subclass of Matter or
Indexer or Counter or any ducktyped class of Matter
strict (bool): True means enforce val in ._ser matches .klas
False means do not enforce. Default False
"""
if klas is None:
from ..core import coring
klas = coring.Matter
super(CesrSuberBase, self).__init__(*pa, **kwa)
self.klas = klas
self.strict = bool(strict)
def _ser(self, val: coring.Matter):
"""
Serialize value to bytes to store in db
When strict is True, val must match .klas or TypeError is raised.
Parameters:
val (coring.Matter): instance Matter ducktype with .qb64b attribute
Returns:
bytes: serialized qb64b bytes suitable for db storage.
Raises:
TypeError: wrong instance class when strict.
"""
if self.strict and not isinstance(val, self.klas):
raise TypeError(f"Expected {self.klas}, got {type(val)}.")
return val.qb64b
def _des(self, val: memoryview | bytes):
"""
Deserialize val to str
Parameters:
val (memoryview | bytes): convertable to coring.matter
"""
if isinstance(val, memoryview): # memoryview is always bytes
val = bytes(val) # convert to bytes
return self.klas(qb64b=val) # qb64b parameter accepts str
[docs]
class CesrSuber(CesrSuberBase, Suber):
"""
Sub class of Suber where data is CESR encode/decode ducktyped subclass
instance such as Matter, Indexer, Counter with .qb64b property when provided
as fully qualified serialization.
Extends Suber to support val that are ducktyped CESR serializable .qb64 .qb64b
subclasses such as coring.Matter, coring.Indexer, coring.Counter.
Automatically serializes and deserializes from qb64b to/from CESR instances
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Type[coring.Matter]): Class reference to subclass of Matter or
Indexer or Counter or any ducktyped class of Matter
"""
super(CesrSuber, self).__init__(*pa, **kwa)
[docs]
class CesrOnSuber(CesrSuberBase, OnSuberBase, Suber):
"""
Subclass of CesrSuberBase, OnSuberBase, and Suber that adds methods for
keys with ordinal numbered tails and values that are Cesr serializations
of Matter subclass ducktypes.
Each key consistes of pre joined with .sep to ordinal suffix
Assumes dupsort==False
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to False
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(CesrOnSuber, self).__init__(*pa, **kwa)
[docs]
class CatCesrSuberBase(CesrSuberBase):
"""Base Class whose values stored in db are a concatenation of the .qb64b property
from one or more subclass instances (qb64b is bytes of fully qualified
serialization) that support CESR encode/decode ducktyped subclass instance
such as Matter, Indexer, Counter
Automatically serializes and deserializes iterable of qb64b to/from CESR instances
._ser override .put .set input value to be instance that is serialized
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
klas (Iterable): of Class references to subclasses of CESR compatible
, each of to Type[coring.Matter etc]
"""
[docs]
def __init__(self,
*pa,
klas: Iterable | Type[coring.Matter] | None = None,
strict: bool = False,
**kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Iterable|Type[coring.Matter]|None): of Class references to
subclasses of CESR compatible Type[coring.Matter etc]Class
reference to subclass of Matter or Indexer or Counter or
any ducktyped class of Matter
None is replaced with default Matter
strict (bool): True means enforce val in ._ser matches .klas
False means do not enforce. Default False
"""
if klas is None:
from ..core import coring
klas = (coring.Matter, ) # set default to tuple of single Matter
if not helping.isNonStringIterable(klas): # not iterable
klas = (klas, ) # make it so
super(CatCesrSuberBase, self).__init__(*pa,
klas=klas,
strict=strict,
**kwa)
def _ser(self, val: Union[Iterable, coring.Matter]):
"""
Serialize val to bytes to store in db
Concatenates .qb64b of each instance in val and returns val bytes
When strict is True, val arity and ordered slot types must match .klas
or ValueError/TypeError is raised.
Parameters:
val (Union[Iterable, coring.Matter]): of subclass instances.
Non-iterables are wrapped as a one-item tuple.
Returns:
bytes: concatenation of serialized qb64b values in order.
Raises:
ValueError: when strict and tuple arity does not match .klas.
TypeError: wrong slot class when strict.
"""
if not helping.isNonStringIterable(val): # not iterable
val = (val, ) # make iterable
vals = tuple(val) if self.strict else val
if self.strict:
for klas, item in zip(self.klas, vals, strict=self.strict):
if not isinstance(item, klas):
raise TypeError(f"Expected {klas}, got {type(item)}.")
return b''.join(obj.qb64b for obj in vals)
def _des(self, val: memoryview | bytes | bytearray):
"""
Converts val bytes to vals tuple of subclass instances by deserializing
.qb64b concatenation in order of each instance in .klas
Returns:
vals (tuple): subclass instances
Parameters:
val (Union[bytes, memoryview]): of concatenation of .qb64b
"""
if not isinstance(val, bytearray): # is memoryview or bytes
val = bytearray(val) # convert so may strip
return tuple(klas(qb64b=val, strip=True) for klas in self.klas)
[docs]
class CatCesrSuber(CatCesrSuberBase, Suber):
"""
Class whose values stored in db are a concatenation of the .qb64b property
from one or more subclass instances (qb64b is bytes of fully qualified
serialization) that support CESR encode/decode ducktyped subclass instance
such as Matter, Indexer, Counter
Automatically serializes and deserializes from qb64b to/from CESR instances
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
klas (Iterable): of Class references to subclasses of CESR compatible
, each of to Type[coring.Matter etc]
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Iterable|Type[coring.Matter]|None): of Class references to
subclasses of CESR compatible Type[coring.Matter etc]Class
reference to subclass of Matter or Indexer or Counter or
any ducktyped class of Matter
None is replaced with default Matter
"""
super(CatCesrSuber, self).__init__(*pa, **kwa)
[docs]
class IoSetSuber(SuberBase):
"""Insertion Ordered Set Suber factory class that supports
a set of distinct entries at a given effective database key but with
dupsort==False. Effective data model is that there are multiple values in a
set of values where every member of the set has the same key (duplicate key).
The set of values is an ordered set using insertion order. Any given value
may appear only once in the set (not a list).
This works similarly to the IO value duplicates for the LMDBer class with a
sub db of LMDB (dupsort==True) but without its size limitation of 511 bytes
for each value when dupsort==True.
Here the key is augmented with a hidden numbered suffix that provides a
an ordered set of values at each effective key (duplicate key). The suffix
is appended and stripped transparently. The set of multiple items with
duplicate keys are retrieved in insertion order when iterating or as a list
of the set elements.
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
"""
[docs]
def __init__(self, db: LMDBer, *,
subkey: str='docs.',
dupsort: bool=False, **kwa):
"""Initialize instance
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Type[coring.Matter]): Class reference to subclass of Matter or
Indexer or Counter or any ducktyped class of Matter
"""
super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa)
[docs]
def put(self, keys: str|bytes|memoryview|Iterable,
vals: str|bytes|memoryview|Iterable|None):
"""Puts all vals at effective key made from keys and hidden ordinal suffix.
that are not already in set of vals at key. Does not overwrite.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
vals (str|bytes|memoryview|NonStrIterable|None): serialized values to add
to set of vals at onkey if any.
When not NonStrIterable converts to iterable.
Empty iterable or None returns False
Returns:
result (bool): True If successful, False otherwise.
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) if vals else () # make iterable
return (self.db.putIoSetVals(db=self.sdb,
key=self._tokey(keys),
vals=[self._ser(val) for val in vals],
sep=self.sep))
[docs]
def pin(self, keys: str|bytes|memoryview|Iterable,
vals: str|bytes|memoryview|Iterable|None):
"""Pins (sets) vals at effective key made from keys and hidden ordinal suffix.
Overwrites. Removes all pre-existing vals that share same effective keys
and replaces them with vals
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
vals (str|bytes|memoryview|Iterable|None): serialized value to replace
Value at onkey.
None means empty iterable.
Empty iterable or None returns False
Returns:
result (bool): True If successful, False otherwise.
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) if vals else () # make iterable
return (self.db.pinIoSetVals(db=self.sdb,
key=self._tokey(keys),
vals=[self._ser(val) for val in vals],
sep=self.sep))
[docs]
def add(self, keys: str|bytes|memoryview|Iterable,
val: str|bytes|memoryview|None):
"""Add val idempotently to vals at effective key made from keys and hidden
ordinal suffix. Idempotent means that added value is not already in set
of vals at key. Does not overwrite or add same value at same key more
than once.
When val None returns False
Parameters:
keys (str|bytes|memoryview|Iterable): of key parts to be
combined in order to form key
val (str|bytes|memoryview|None): value to add
When val None returns False
Returns:
result (bool): True means unique value added among duplications,
False means duplicate of same value already exists.
"""
return (self.db.addIoSetVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val),
sep=self.sep))
[docs]
def getItem(self, keys: str|bytes|memoryview|Iterable, *, ion=0):
"""Gets item list in set at effective key made from keys and hidden
ordinal suffix ion starting at ion >= ion.
When keys is empty or missing then returns empty list
All vals in set of vals that share same effecive key are retrieved in
insertion order starting at ion.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
ion (int): offset into set to start the count (0 based offset)
Returns:
vals (list[str]): each item in list is str
empty list if no entry at keys
"""
# use iter so can more efficiently ._des(val)
return [(self._tokeys(key), self._des(val)) for key, val in
self.db.getIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
ion=ion,
sep=self.sep)]
[docs]
def get(self, keys: str|bytes|memoryview|Iterable, *, ion=0):
"""Gets vals set list at effective key made from keys and hidden
ordinal suffix ion starting at ion >= ion.
When keys is empty or missing then returns empty list
All vals in set of vals that share same effecive key are retrieved in
insertion order starting at ion.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
ion (int): offset into set to start the count (0 based offset)
Returns:
vals (list[str]): each item in list is str
empty list if no entry at keys
"""
# use iter so can more efficiently ._des(val)
return [self._des(val) for key, val in
self.db.getIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
ion=ion,
sep=self.sep)]
[docs]
def getItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0):
"""Iterates over set items at effecive key made from keys and hidden
ordinal suffix ion starting at ion >= ion.
When keys is empty or missing then returns empty iterator
All vals in set of vals that share same effecive key are retrieved in
insertion order starting at ion.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
ion (int): offset into set to start the count (0 based offset)
Returns:
items (Iterator[str]): entries in set at key for ion >= ion.
Raises StopIteration when done
"""
for key, val in self.db.getIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
ion=ion,
sep=self.sep):
yield (self._tokeys(key), self._des(val))
[docs]
def getIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0):
"""Iterates over set values at effecive key made from keys and hidden
ordinal suffix ion starting at ion >= ion.
When keys is empty or missing then returns empty iterator
All vals in set of vals that share same effecive key are retrieved in
insertion order starting at ion.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
ion (int): offset into set to start the count (0 based offset)
Returns:
vals (Iterator[str]): str values. Raises StopIteration when done
"""
for key, val in self.db.getIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
ion=ion,
sep=self.sep):
yield self._des(val)
[docs]
def getLastItem(self, keys: str|bytes|memoryview|Iterable):
"""Gets last set item (key, val) inserted at effecive key where
effective key is made from keys and hidden ordinal suffix
when keys is empty or missing returns empty tuple.
All items in the set of items that shares the same effecive key are
retrieved in insertion order.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
Returns:
last ((str, str)|None): (key, val) tuple or empty tuple if no entry
at keys or keys is empty
"""
if last := self.db.getIoSetLastItem(db=self.sdb, key=self._tokey(keys)):
key, val = last
return (self._tokeys(key), self._des(val))
return last
[docs]
def getLast(self, keys: str|bytes|memoryview|Iterable):
"""Gets last set val inserted at effecive key where effective key is
made from keys and hidden ordinal suffix.
All vals in the set of vals that shares the same effecive key are retrieved in
insertion order.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
Returns:
val (str): value str, None if no entry at keys
"""
if last := self.db.getIoSetLastItem(db=self.sdb, key=self._tokey(keys)):
key, val = last
return self._des(val)
return None
[docs]
def rem(self, keys: str|bytes|memoryview|Iterable,
val: str|bytes|memoryview|None=None):
"""Removes entry at effective key made from keys and hidden ordinal suffix
that matches val if any.
When val is None, default, then removes all entries at keys
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined in
order to form key
val (str|bytes|memoryview|None): value at key to delete. Subclass ._ser
method may accept different value types
if val is None then remove all values at key
Returns:
result (bool): True if effective key with val exists so rem successful.
False otherwise
"""
return self.db.remIoSetVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val) if val is not None else val,
sep=self.sep)
[docs]
def cnt(self, keys: str|bytes|memoryview|Iterable="", *, ion=0):
"""Counts entries at effective key made from keys and hidden ordinal
suffix. Zero otherwise.
When keys empty then counts all in db.
Returns:
cnt (int): entries in set at keys starting with ion >= ion.
When keys is empty then counts all entries in db.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key
ion (int): offset into set to start the count (0 based offset)
"""
if not keys:
return self.db.cntAll(db=self.sdb)
return (self.db.cntIoSet(db=self.sdb,
key=self._tokey(keys),
ion=ion,
sep=self.sep))
[docs]
def getTopItemIter(self, keys: str|bytes|memoryview|Iterable = "",
*, topive=False):
"""Iterates over all the items in top branch defined by keys where
keys may be truncation of full branch.
Transparently suffix and unsuffix insertion ordering ordinal
raises StopIterationError when done
Returns:
items (Iterator): of (key, val) tuples over the all the items in
subdb whose effective key startswith key made from keys.
Keys may be keyspace prefix in order to return branches of key space.
When keys is empty then returns all items in subdb.
Returned key in each item has ordinal suffix removed.
Parameters:
keys (Iterable): tuple of bytes or strs that may be a truncation of
a full keys tuple in in order to address all the items from
multiple branches of the key space. If keys is empty then gets
all items in database.
Either append "" to end of keys Iterable to ensure get properly
separated top branch key or use top=True.
In Python str.startswith('') always returns True so if branch
key is empty string it matches all keys in db with startswith.
topive (bool): True means treat as partial key tuple from top branch of
key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of top value
Uses python .startswith to match which always returns True if top is
empty string so empty will matches all keys in db.
"""
for key, val in self.db.getTopIoSetItemIter(db=self.sdb,
top=self._tokey(keys, topive=topive), sep=self.sep.encode()):
yield (self._tokeys(key), self._des(val))
[docs]
def getLastIter(self, keys: str|bytes|memoryview|Iterable = ""):
"""Iterates over last val inserted in each set starting at key made
from keys for all keys in db where key >= key.
Each effective key is made from keys and hidden ordinal suffix.
All vals in the set of vals that shares the same effecive key are retrieved in
insertion order.
raises StopIterationError when done
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key. Iterates over all keys when key empty.
Returns:
vals (Iterator[str]): value str
"""
for val in self.db.getIoSetLastIterAll(db=self.sdb, key=self._tokey(keys)):
yield self._des(val)
[docs]
def getLastItemIter(self, keys: str|bytes|memoryview|Iterable = ""):
"""Iterates over last item inserted in each set starting at key made
from keys for all keys in db where key >= key.
Each effective key is made from keys and hidden ordinal suffix.
All vals in the set of vals that shares the same effecive key are retrieved in
insertion order.
raises StopIterationError when done
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be combined
in order to form key. Iterates over all keys when key empty.
Returns:
items (Iterator[(str, str)]): (key, val) tuples
"""
for key, val in self.db.getIoSetLastItemIterAll(db=self.sdb, key=self._tokey(keys)):
yield (self._tokeys(key), self._des(val))
[docs]
class B64IoSetSuber(B64SuberBase, IoSetSuber):
"""Subclass of B64SuberBase and IoSetSuber that serializes and deserializes
values as .sep joined strings of Base64 components in insertion order using
hidden ion suffix in keyspace. Using IoSet removes 511 byte limitation of
duplicates (dupsort=True).
Assumes dupsort==False
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to False
sep (str): separator for combining keys tuple of strs into key bytes
for db key and also used to convert val iterator to val bytes
Must not be Base64 character.
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(B64IoSetSuber, self).__init__(*pa, **kwa)
[docs]
class CesrIoSetSuber(CesrSuberBase, IoSetSuber):
"""
Subclass of CesrSuber and IoSetSuber.
Sub class of Suber where data is CESR encode/decode ducktyped subclass
instance such as Matter, Indexer, Counter with .qb64b property when provided
as fully qualified serialization
Automatically serializes and deserializes from qb64b to/from CESR instances
Extends IoSetSuber with mixin methods ._ser and ._des from CesrSuberBase
so that all IoSetSuber methods now work with CESR subclass for each val.
IoSetSuber stores at each effective key a set of distinct values that
share that same effective key where each member of the set is retrieved in
insertion order (dupsort==False)
The methods allows an Iterable (set valued) of Iterables of Matter subclass
instances to be stored at a given effective key in insertion order.
Actual keys include a hidden ordinal key suffix that tracks insertion order.
The suffix is appended and stripped transparently from the keys. The set of
items with duplicate effective keys are retrieved in insertion order when
iterating or as a list of the set elements. The actual iokey for any item
includes the ordinal suffix.
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
klas (Iterable): of Class references to subclasses of CESR compatible
, each of to Type[coring.Matter etc]
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Type[coring.Matter]): Class reference to subclass of Matter or
Indexer or Counter or any ducktyped class of Matter
"""
super(CesrIoSetSuber, self).__init__(*pa, **kwa)
[docs]
class CatCesrIoSetSuber(CatCesrSuberBase, IoSetSuber):
"""
Sub class of CatSuberBase and IoSetSuber where values stored in db are a
concatenation of .qb64b property from one or more Cesr compatible subclass
instances that automatically serializes and deserializes to/from qb64b .
(qb64b is bytes of fully qualified serialization).
Extends IoSetSuber with mixin methods ._ser and ._des from CatSuberBase
so that all IoSetSuber methods now work with an Iterable of CESR subclass
for each val.
IoSetSuber stores at each effective key a set of distinct values that
share that same effective key where each member of the set is retrieved in
insertion order (dupsort==False)
The methods allows an Iterable (set valued) of Iterables of Matter subclass
instances to be stored at a given effective key in insertion order.
Actual keys include a hidden ordinal key suffix that tracks insertion order.
The suffix is appended and stripped transparently from the keys. The set of
items with duplicate effective keys are retrieved in insertion order when
iterating or as a list of the set elements. The actual iokey for any item
includes the ordinal suffix.
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
klas (Iterable): of Class references to subclasses of Matter, each
of to Type[coring.Matter]
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Iterable|Type[coring.Matter]|None): of Class references to
subclasses of CESR compatible Type[coring.Matter etc]Class
reference to subclass of Matter or Indexer or Counter or
any ducktyped class of Matter
None is replaced with default Matter
"""
super(CatCesrIoSetSuber, self).__init__(*pa, **kwa)
[docs]
class SignerSuber(CesrSuber):
"""Sub class of CesrSuber where data is Signer subclass instance .qb64b propery
which is a fully qualified serialization and uses the key which is the qb64b
of the signer.verfer to get the transferable property of the verfer
Automatically serializes and deserializes from qb64b to/from Signer instances
Assumes that last or only element of db key from keys for all entries is the qb64
of a public key for the associated Verfer instance. This allows returned
Signer instance to have its .transferable property set correctly.
"""
[docs]
def __init__(self, *pa, klas: Type[signing.Signer] | None = None, **kwa):
"""
Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
klas (Type[coring.Matter]): Class reference to subclass of Matter
"""
from ..core import signing
if klas is None:
klas = signing.Signer
if not (issubclass(klas, signing.Signer)):
raise ValueError("Invalid klas type={}, expected {}."
"".format(klas, signing.Signer))
super(SignerSuber, self).__init__(*pa, **kwa)
self.klas = klas
[docs]
def get(self, keys: Union[str, Iterable]):
"""
Gets Signer instance at keys
Returns:
val (Signer): transferable determined by key which is verfer
None if no entry at keys
Parameters:
keys (Union[str, iterable]): key strs to be combined in order to
form key. Last element of keys is verkey used to determin
.transferable for Signer
Usage:
Use walrus operator to catch and raise missing entry
if (signer := mydb.get(keys)) is None:
raise ExceptionHere
use signer here
"""
key = self._tokey(keys) # keys maybe string or tuple
val = self.db.getVal(db=self.sdb, key=key)
keys = self._tokeys(key) # verkey is last split if any
from ..core import coring
verfer = coring.Verfer(qb64b=keys[-1]) # last split
return (self.klas(qb64b=bytes(val), transferable=verfer.transferable)
if val is not None else None)
[docs]
def getTopItemIter(self, keys: str | bytes | memoryview | Iterable = "",
*, topive=False):
"""Iterates over all the items in top branch defined by keys where
keys may be truncation of full branch.
Returns:
iterator (Iteratore: tuple (key, val) over the all the items in
subdb whose key startswith key made from keys. Keys may be keyspace
prefix to return branches of key space. When keys is empty then
returns all items in subdb
Parameters:
keys (Iterable): tuple of bytes or strs that may be a truncation of
a full keys tuple in in order to get all the items from
multiple branches of the key space. If keys is empty then gets
all items in database.
topive (bool): True means treat as partial key tuple from top branch of
key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of top value
Uses python .startswith to match which always returns True if top is
empty string so empty will matches all keys in db.
"""
for key, val in self.db.getTopItemIter(db=self.sdb,
top=self._tokey(keys, topive=topive)):
ikeys = self._tokeys(key) # verkey is last split if any
from ..core import coring
verfer = coring.Verfer(qb64b=ikeys[-1]) # last split
yield (ikeys, self.klas(qb64b=bytes(val),
transferable=verfer.transferable))
[docs]
class CryptSignerSuber(SignerSuber):
"""
Sub class of SignerSuber where data is Signer subclass instance .qb64b property
that has been encrypted if encrypter provided.
which is a fully qualified serialization and uses the key which is the qb64b
of the signer.verfer to get the transferable property of the verfer
Automatically serializes and deserializes from qb64b to/from Signer instances
Assumes that last or only element of db key from keys for all entries is the qb64
of a public key for the associated Verfer instance. This allows returned
Signer instance to have its .transferable property set correctly.
"""
[docs]
def put(self, keys: Union[str, Iterable], val: coring.Matter,
encrypter: signing.Encrypter = None):
"""
Puts qb64 of Matter instance val at key made from keys. Does not overwrite
If encrypter provided then encrypts first
Parameters:
keys (tuple): of key strs to be combined in order to form key
val (Signer): instance of self.klas
encrypter (signing.Encrypter): optional
Returns:
result (bool): True If successful, False otherwise, such as key
already in database.
"""
if encrypter:
val = encrypter.encrypt(prim=val) # returns Cipher instance
return (self.db.putVal(db=self.sdb,
key=self._tokey(keys),
val=val.qb64b))
[docs]
def pin(self, keys: Union[str, Iterable], val: coring.Matter,
encrypter: signing.Encrypter = None):
"""
Pins (sets) qb64 of Matter instance val at key made from keys. Overwrites.
If encrypter provided then encrypts first
Parameters:
keys (tuple): of key strs to be combined in order to form key
val (Signer): instance of self.klas
encrypter (signing.Encrypter): optional
Returns:
result (bool): True If successful. False otherwise.
"""
if encrypter:
val = encrypter.encrypt(prim=val) # returns Cipher instance
return (self.db.setVal(db=self.sdb,
key=self._tokey(keys),
val=val.qb64b))
[docs]
def get(self, keys: Union[str, Iterable], decrypter: signing.Decrypter = None):
"""
Gets Signer instance at keys. If decrypter then assumes value in db was
encrypted and so decrypts value in db before converting to Signer.
Returns:
val (Signer): transferable determined by key which is verfer
None if no entry at keys
Parameters:
keys (Union[str, iterable]): key strs to be combined in order to
form key. Last element of keys is verkey used to determin
.transferable for Signer
decrypter (signing.Decrypter): optional. If provided assumes value in
db was encrypted and so decrypts before converting to Signer.
Usage:
Use walrus operator to catch and raise missing entry
if (signer := mydb.get(keys)) is None:
raise ExceptionHere
use signer here
"""
key = self._tokey(keys) # keys maybe string or tuple
val = self.db.getVal(db=self.sdb, key=key)
if val is None:
return None
keys = self._tokeys(key) # verkey is last split if any
from ..core import coring
verfer = coring.Verfer(qb64b=keys[-1]) # last split
if decrypter:
return (decrypter.decrypt(qb64=bytes(val),
transferable=verfer.transferable))
return (self.klas(qb64b=bytes(val), transferable=verfer.transferable))
[docs]
def getTopItemIter(self, keys: str|bytes|memoryview|Iterable= "",
decrypter: signing.Decrypter = None, *, topive=False):
"""Iterates over all the items in top branch defined by keys where
keys may be truncation of full branch.
Returns:
items (Iterator): of tuples (key, val) over the all the items in
subdb whose key startswith key made from keys. Keys may be keyspace
prefix to return branches of key space. When keys is empty then
returns all items in subdb
decrypter (signing.Decrypter): optional. If provided assumes value in
db was encrypted and so decrypts before converting to Signer.
Parameters:
keys (Iterable): tuple of bytes or strs that may be a truncation of
a full keys tuple in in order to get all the items from
multiple branches of the key space. If keys is empty then gets
all items in database.
topive (bool): True means treat as partial key tuple from top branch of
key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of top value
Uses python .startswith to match which always returns True if top is
empty string so empty will matches all keys in db.
"""
for key, val in self.db.getTopItemIter(db=self.sdb,
top=self._tokey(keys, topive=topive)):
ikeys = self._tokeys(key) # verkey is last split if any
from ..core import coring
verfer = coring.Verfer(qb64b=ikeys[-1]) # last split
if decrypter:
yield (ikeys, decrypter.decrypt(qb64=bytes(val),
transferable=verfer.transferable))
else:
yield (ikeys, self.klas(qb64b=bytes(val),
transferable=verfer.transferable))
[docs]
class SerderSuberBase(SuberBase):
"""
Sub class of SuberBase where data is serialized Serder Subclass instance
given by .klas
Automatically serializes and deserializes using .klas Serder methods
"""
[docs]
def __init__(self, *pa,
klas: Type[serdering.Serder] | None = None,
**kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
Overridden Parameters:
klas (Type[serdering.Serder]): Class reference to subclass of Serder
"""
if klas is None:
from ..core import serdering
klas = serdering.SerderKERI
super(SerderSuberBase, self).__init__(*pa, **kwa)
self.klas = klas
def _ser(self, val: serdering.Serder):
"""
Serialize value to bytes to store in db
Parameters:
val (serdering.Serder): instance Serder subclass like SerderKERI
"""
return val.raw
def _des(self, val: (str | memoryview | bytes)):
"""
Deserialize val to str
Parameters:
val (Union[str, memoryview, bytes]): convertable to coring.matter
"""
if isinstance(val, memoryview): # memoryview is always bytes
val = bytes(val) # convert to bytes
elif hasattr(val, "encode"): # str
val = val.encode() # convert to bytes
return self.klas(raw=val, verify=self.verify)
[docs]
class SerderSuber(SerderSuberBase, Suber):
"""
Sub class of SerderSuberBase, Suber where data is serialized Serder Subclass
instance given by .klas
Automatically serializes and deserializes using .klas Serder methods
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Type[serdering.Serder]): Class reference to subclass of Serder
"""
super(SerderSuber, self).__init__(*pa, **kwa)
[docs]
class SerderIoSetSuber(SerderSuberBase, IoSetSuber):
"""
Sub class of SerderSuberBase and IoSetSuber that allows multiple Serder
instances to be stored at the same db key in insertion order.
Example use case would be an escrow where the key is a sequence number
based index (such as snKey).
Sub class of SerderSuberBase where data is serialized Serder Subclass instance
given by .klas
Automatically serializes and deserializes using .klas Serder methods
Extends IoSetSuber so that all IoSetSuber methods now work with Serder
subclass for each val.
IoSetSuber stores at each effective key a set of distinct values that
share that same effective key where each member of the set is retrieved in
insertion order (dupsort==False)
The methods allows an Iterable (set valued) of Iterables of separation subclass
instances to be stored at a given effective key in insertion order.
Actual keys include a hidden ordinal key suffix that tracks insertion order.
The suffix is appended and stripped transparently from the keys. The set of
items with duplicate effective keys are retrieved in insertion order when
iterating or as a list of the set elements. The actual iokey for any item
includes the ordinal suffix.
Attributes:
db (LMDBer): base LMDB db
sdb (lmdb._Database): instance of lmdb named sub db for this Suber
sep (str): separator for combining keys tuple of strs into key bytes
klas (Iterable): of Class references to subclasses of CESR compatible
, each of to Type[coring.Matter etc]
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Type[serdering.Serder]): Class reference to subclass of Serder
"""
super(SerderIoSetSuber, self).__init__(*pa, **kwa)
[docs]
class SchemerSuber(SerderSuberBase, Suber):
"""
Sub class of SerderSuberBase and Suber where data is serialized Schemer instance
Schemer ser/des is ducktype of Serder using .raw
Automatically serializes and deserializes using Schemer methods
"""
[docs]
def __init__(self, *pa,
klas: Type[scheming.Schemer] | None = None,
**kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Type[scheming.Schemer]): Class reference to ducktyped subclass
of Serder
Overridden Parameters:
klas (Type[scheming.Schemer]): Class reference to ducktyped subclass
of Serder intercepts passed in klas and forces it to Schemer
"""
from ..core import scheming
if klas is None:
klas = scheming.Schemer
if not issubclass(klas, scheming.Schemer):
raise TypeError(f"Invalid {klas=}, not subclass of {scheming.Schemer}.")
super(SchemerSuber, self).__init__(*pa, klas=klas, **kwa)
[docs]
class DupSuber(SuberBase):
"""
Sub DB of LMDBer. Subclass of SuberBase that supports multiple entries at
each key (duplicates) with dupsort==True
Do not use if serialized value is greater than 511 bytes.
This is a limitation of dupsort==True sub dbs in LMDB
"""
[docs]
def __init__(self, db: Type[LMDBer], *,
subkey: str='docs.',
dupsort: bool=True,
**kwa):
"""
Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True (forced default) means enable duplicates at each key
False means do not enable duplicates at each key
"""
super(DupSuber, self).__init__(db=db, subkey=subkey, dupsort=True, **kwa)
[docs]
def put(self, keys: str | bytes | memoryview | Iterable,
vals: str | bytes | memoryview | Iterable):
"""
Puts all vals at key made from keys. Does not overwrite. Adds to existing
dup values at key if any. Duplicate means another entry at the same key
but the entry is still a unique value. Duplicates are inserted in
lexocographic order not insertion order. Lmdb does not insert a duplicate
unless it is a unique value for that key.
Parameters:
keys (str | bytes | memoryview | Iterable): of key strs to be
combined in order to form key
vals (str | bytes | memoryview | Iterable): str or bytes of each
value to be written at key
Returns:
result (bool): True If successful, False otherwise.
Apparently always returns True (how .put works with dupsort=True)
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) # make iterable
return (self.db.putVals(db=self.sdb,
key=self._tokey(keys),
vals=[self._ser(val) for val in vals]))
[docs]
def pin(self, keys: str | bytes | memoryview | Iterable,
vals: str | bytes | memoryview | Iterable):
"""
Pins (sets) vals at key made from keys. Overwrites. Removes all
pre-existing dup vals and replaces them with vals
Parameters:
keys (str | bytes | memoryview | Iterable): of key strs to be
combined in order to form key
vals (str | bytes | memoryview | Iterable): str or bytes values
Returns:
result (bool): True If successful, False otherwise.
"""
key = self._tokey(keys)
self.db.delVals(db=self.sdb, key=key) # delete all values
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) # make iterable
return (self.db.putVals(db=self.sdb,
key=key,
vals=[self._ser(val) for val in vals]))
[docs]
def add(self, keys: str | bytes | memoryview | Iterable,
val: str | bytes | memoryview ):
"""
Add val to vals at key made from keys. Does not overwrite. Adds to existing
dup values at key if any. Duplicate means another entry at the same key
but the entry is still a unique value. Duplicates are inserted in
lexocographic order not insertion order. Lmdb does not insert a duplicate
unless it is a unique value for that key.
Parameters:
keys (str | bytes | memoryview | Iterable): of key strs to be combined in order to form key
val (str | bytes | memoryview): value
Returns:
result (bool): True means unique value among duplications,
False means duplicte of same value already exists.
"""
return (self.db.addVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val)))
[docs]
def get(self, keys: str | bytes | memoryview | Iterable):
"""
Gets dup vals list at key made from keys
Parameters:
keys (str | bytes | memoryview | Iterable): of key strs to be
combined in order to form key
Returns:
vals (list): each item in list is str
empty list if no entry at keys
"""
return [self._des(val) for val in
self.db.getValsIter(db=self.sdb, key=self._tokey(keys))]
[docs]
def getLast(self, keys: str | bytes | memoryview | Iterable):
"""
Gets last dup val at key made from keys
Parameters:
keys (tuple): of key strs to be combined in order to form key
Returns:
val (str): value else None if no value at key
"""
val = self.db.getValLast(db=self.sdb, key=self._tokey(keys))
return self._des(val) if val is not None else val
[docs]
def getIter(self, keys: str | bytes | memoryview | Iterable):
"""
Gets dup vals iterator at key made from keys
Duplicates are retrieved in lexocographic order not insertion order.
Parameters:
keys (str | bytes | memoryview | Iterable): of key strs to be
combined in order to form key
Returns:
iterator: vals each of str. Raises StopIteration when done
"""
for val in self.db.getValsIter(db=self.sdb, key=self._tokey(keys)):
yield self._des(val)
[docs]
def cnt(self, keys: str|bytes|memoryview|Iterable = ""):
"""Counts dup values at key made from keys, zero otherwise
When keys is empty then counts all in db.
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be
combined in order to form key. When keys empty then count all
entires in db.
"""
if not keys:
return self.db.cntAll(db=self.sdb)
return (self.db.cntVals(db=self.sdb, key=self._tokey(keys)))
[docs]
def rem(self, keys: str | bytes | memoryview | Iterable,
val: str|bytes|memoryview|None=None):
"""Removes entry at keys
Parameters:
keys (tuple): of key strs to be combined in order to form key
val (str|bytes|memoryview|None): instance of dup val at key to delete
if val is None, default, then remove all values at key
Returns:
result (bool): True if key exists so delete successful. False otherwise
"""
if val is None:
return (self.db.delVals(db=self.sdb,
key=self._tokey(keys)))
return (self.db.delVals(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val)))
[docs]
class CesrDupSuber(CesrSuberBase, DupSuber):
"""
Sub class of DupSuber whose values are CESR ducktypes of Matter subclasses.
serialized to and deserialied from val instance .qb64b property
which is a fully qualified serialization.
Automatically serializes and deserializes from qb64b to/from Matter ducktyped
instances
DupSuber supports multiple entries at each key (duplicates) with dupsort==True
Do not use if serialized value is greater than 511 bytes.
This is a limitation of dupsort==True sub dbs in LMDB
"""
[docs]
def __init__(self, *pa, **kwa):
"""Initialize Instance
Inherited Parameters:
"""
super(CesrDupSuber, self).__init__(*pa, **kwa)
[docs]
class CatCesrDupSuber(CatCesrSuberBase, DupSuber):
"""
Sub class of DupSuber whose values are CESR ducktypes of Matter subclasses.
serialized to and deserialied from val instance .qb64b property
which is a fully qualified serialization.
Automatically serializes and deserializes from qb64b to/from Matter ducktyped
instances
DupSuber supports multiple entries at each key (duplicates) with dupsort==True
Do not use if serialized value is greater than 511 bytes.
This is a limitation of dupsort==True sub dbs in LMDB
"""
[docs]
def __init__(self, *pa, **kwa):
"""Initialize Instance
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True (forced default) means enable duplicates at each key
False means do not enable duplicates at each key
sep (str): separator to convert keys iterator to key bytes for db key
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
klas (Iterable|Type[coring.Matter]|None): of Class references to
subclasses of CESR compatible Type[coring.Matter etc]Class
reference to subclass of Matter or Indexer or Counter or
any ducktyped class of Matter
None is replaced with default Matter
"""
super(CatCesrDupSuber, self).__init__(*pa, **kwa)
[docs]
class IoDupSuber(DupSuber):
"""
Sub class of DupSuber that supports Insertion Ordering (IoDup) of duplicates
By automagically prepending and stripping ordinal proem to/from each
duplicate value at a given key.
IoDupSuber supports insertion ordered multiple entries at each key
(duplicates) with dupsort==True
Do not use if serialized length key + proem + value, is greater than 511 bytes.
This is a limitation of dupsort==True sub dbs in LMDB
IoDupSuber may be more performant then IoSetSuber for values that are indices
to other sub dbs that fit the size constraint because LMDB support for
duplicates is more space efficient and code performant.
Duplicates at a given key preserve insertion order of duplicate.
Because lmdb is lexocographic an insertion ordering proem is prepended to
all values that makes lexocographic order that same as insertion order.
Duplicates are ordered as a pair of key plus value so prepending proem
to each value changes duplicate ordering. Proem is 33 characters long.
With 32 character hex string followed by '.' for essentiall unlimited
number of values which will be limited by memory.
With prepended proem ordinal must explicity check for duplicate values
before insertion. Uses a python set for the duplicate inclusion test.
Set inclusion scales with O(1) whereas list inclusion scales with O(n).
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
"""
super(IoDupSuber, self).__init__(*pa, **kwa)
[docs]
def put(self, keys: str | bytes | memoryview | Iterable,
vals: str | bytes | memoryview | Iterable):
"""Puts all vals idempotently at key made from keys in insertion order using
hidden ordinal proem. Idempotently means do not put any val in vals that is
already in dup vals at key. Does not overwrite.
Parameters:
keys (Iterable): of key strs to be combined in order to form key
vals (Iterable): of str serializations
Returns:
result (bool): True If successful, False otherwise.
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) # make iterable
return (self.db.putIoDupVals(db=self.sdb,
key=self._tokey(keys),
vals=[self._ser(val) for val in vals]))
[docs]
def pin(self, keys: str | bytes | memoryview | Iterable,
vals: str | bytes | memoryview | Iterable):
"""Pins (sets) vals at key made from keys in insertion order using hidden
ordinal proem. Overwrites. Removes all pre-existing vals that share
same keys and replaces them with vals
Parameters:
keys (Iterable): of key strs to be combined in order to form key
vals (Iterable): str serializations
Returns:
result (bool): True If successful, False otherwise.
"""
key = self._tokey(keys)
self.db.delIoDupVals(db=self.sdb, key=key) # delete all values
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) # make iterable
return self.db.putIoDupVals(db=self.sdb,
key=key,
vals=[self._ser(val) for val in vals])
[docs]
def add(self, keys: str | bytes | memoryview | Iterable,
val: str | bytes | memoryview):
"""
Add val idempotently at key made from keys in insertion order using hidden
ordinal proem. Idempotently means do not add val that is already in
dup vals at key. Does not overwrite.
Parameters:
keys (Iterable): of key strs to be combined in order to form key
val (str | bytes | memoryview): serialization
Returns:
result (bool): True means unique value added among duplications,
False means duplicate of same value already exists.
"""
return (self.db.addIoDupVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val)))
[docs]
def get(self, keys: str | bytes | memoryview | Iterable):
"""
Gets vals dup list in insertion order using key made from keys and
hidden ordinal proem on dups.
Parameters:
keys (Iterable): of key strs to be combined in order to form key
Returns:
vals (Iterable): each item in list is str
empty list if no entry at keys
"""
return ([self._des(val) for val in
self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))])
[docs]
def getItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0):
"""
Gets vals dup iterator in insertion order using key made from keys and
hidden ordinal proem on dups.
All vals in dups that share same key are retrieved in insertion order.
Parameters:
keys (str | bytes | memoryview | Iterable): of key parts
ion (int): offset into set to start the count (0 based offset)
Returns:
items (Iterator[str]): entries in dups at key for ion >= ion.
Raises StopIteration when done
"""
for key, val in self.db.getIoDupItemIter(db=self.sdb,
key=self._tokey(keys),
ion=ion):
yield (self._tokeys(key), self._des(val))
[docs]
def getIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0):
"""
Gets vals dup iterator in insertion order using key made from keys and
hidden ordinal proem on dups.
All vals in dups that share same key are retrieved in insertion order.
Parameters:
keys (str | bytes | memoryview | Iterable): of key parts
ion (int): offset into set to start the count (0 based offset)
Returns:
vals (Iterator): str values. Raises StopIteration when done
"""
for key, val in self.db.getIoDupItemIter(db=self.sdb,
key=self._tokey(keys),
ion=ion):
yield self._des(val)
[docs]
def getLast(self, keys: str | bytes | memoryview | Iterable):
"""
Gets last val inserted at key made from keys in insertion order using
hidden ordinal proem.
Parameters:
keys (Iterable): of key strs to be combined in order to form key
Returns:
val (str): value str, None if no entry at keys
"""
val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys))
return (self._des(val) if val is not None else val)
[docs]
def rem(self, keys: str|bytes|memoryview|Iterable,
val: str|bytes|memoryview|None=None):
"""Removes entry at key made from keys and dup val that matches val if any,
notwithstanding hidden ordinal proem.
If val is None, default removes all dup values at key if any.
Parameters:
keys (str | bytes | memoryview | Iterable): of key parts to be
combined in order to form key
val (str|bytes|memoryview|None): value at key to delete.
Subclass ._ser method may accept different value types
if val is None then remove all values at key
Returns:
result (bool): True if key with dup val exists so rem successful.
False otherwise
"""
if val is None:
return self.db.delIoDupVals(db=self.sdb, key=self._tokey(keys))
return self.db.delIoDupVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val))
[docs]
def cnt(self, keys: str|bytes|memoryview|Iterable = ""):
"""Counts dup values at key made from keys with hidden ordinal
proem. Zero otherwise. When keys empty then counts All entries in db
not just those at a given key
Parameters:
keys (str|bytes|memoryview|Iterable): of key parts to be combined
in order to form key. When empty counts all entries in db.
"""
if not keys:
return self.db.cntAll(db=self.sdb)
return (self.db.cntIoDups(db=self.sdb, key=self._tokey(keys)))
[docs]
def getTopItemIter(self, keys: str | bytes | memoryview | Iterable = "",
*, topive=False):
"""Iterates over all the items in top branch defined by keys where
keys may be truncation of full branch.
Transparently prepends and strips insertion ordering proem from value
Return iterator over all the items including dup items for all keys
in top branch defined by keys where keys may be truncation of full branch.
Returns:
items (Iterator): of (key, val) tuples over the all the items in
subdb whose key startswith key made from keys and val has its hidden
dup ordinal proem removed.
Keys may be keyspace prefix in order to return branches of key space.
When keys is empty then returns all items in subdb.
Parameters:
keys (str | bytes | memoryview | Iterable): key or key parts that
may be a truncation of a full keys tuple in in order to address
all the items from multiple branches of the key space.
If keys is empty then gets all items in database.
Either append "" to end of keys Iterable to ensure get properly
separated top branch key or use top=True.
In Python str.startswith('') always returns True so if branch
key is empty string it matches all keys in db with startswith.
topive (bool): True means treat as partial key tuple from top branch of
key space given by partial keys. Resultant key ends in .sep
character.
False means treat as full branch in key space. Resultant key
does not end in .sep character.
When last item in keys is empty str then will treat as
partial ending in sep regardless of top value
Uses python .startswith to match which always returns True if top is
empty string so empty will matches all keys in db.
"""
for key, val in self.db.getTopIoDupItemIter(db=self.sdb,
top=self._tokey(keys, topive=topive)):
yield (self._tokeys(key), self._des(val))
[docs]
class B64IoDupSuber(B64SuberBase, IoDupSuber):
"""
Subclass of B64SuberBase and IoDupSuber that serializes and deserializes
values as .sep joined strings of Base64 components in insertion ordered
duplicates with leading value proem. Proem + .sep joined value and must fit
in 511 bytes of keyspace as duplicate
Assumes dupsort==True
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to True
sep (str): separator for combining keys tuple of strs into key bytes
for db key and also used to convert val iterator to val bytes
Must not be Base64 character.
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(B64IoDupSuber, self).__init__(*pa, **kwa)
[docs]
class OnIoDupSuber(OnSuberBase, IoDupSuber):
"""
Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering
(IoDup) of duplicates but where the trailing part of the key space is
a serialized monotonically increasing ordinal number. This is useful for
escrows of key events which are ordinally numbered such as sn but where
duplicates of likely events are also maintained in insertion order.
Insertion order is maintained by automagically prepending and stripping an
ordinal ordering proem to/from each duplicate value at a given key.
OnIoDupSuber adds the convenience methods from OnSuberBase to IoDupSuber for
those cases where the keyspace has a trailing ordinal part.
There are two ordinals, one in the key space and a hidden one in the duplicate
data value space.
OnIoDupSuber supports insertion ordered multiple entries at each key
(duplicates) with dupsort==True
Do not use if serialized length key + proem + value, is greater than 511 bytes.
This is a limitation of dupsort==True sub dbs in LMDB
OnIoDupSuber may be more performant then IoSetSuber for values that are indices
to other sub dbs that fit the size constraint because LMDB support for
duplicates is more space efficient and code performant.
Duplicates at a given key preserve insertion order of duplicate.
Because lmdb is lexocographic an insertion ordering proem is prepended to
all values that makes lexocographic order that same as insertion order.
Duplicates are ordered as a pair of key plus value so prepending proem
to each value changes duplicate ordering. Proem is 33 characters long.
With 32 character hex string followed by '.' for essentiall unlimited
number of values which will be limited by memory.
With prepended proem ordinal must explicity check for duplicate values
before insertion. Uses a python set for the duplicate inclusion test.
Set inclusion scales with O(1) whereas list inclusion scales with O(n).
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
"""
super(OnIoDupSuber, self).__init__(*pa, **kwa)
[docs]
def put(self, keys: str|bytes|memoryview|Iterable, on: int=0,
vals: str|bytes|memoryview|Iterable = b""):
"""Put all vals idempotently at key at key made from keys with on suffix
in insertion order using hidden ordinal proem. Idempotently means do
not put any val in vals that is already in dup vals at key. Does not overwrite.
Parameters:
keys (Iterable): of key strs to be combined in order to form key
on (int): ordinal number used with onKey(pre,on) to form onkey.
vals (Iterable): of str serializations
Returns:
result (bool): True If successful, False otherwise.
"""
if not helping.isNonStringIterable(vals): # not NonStrIterable
vals = (vals, ) if vals else () # make iterable
return self.db.putOnIoDupVals(db=self.sdb,
key=self._tokey(keys),
on=on,
vals=tuple(self._ser(val) for val in vals),
sep=self.sep.encode())
[docs]
def pin(self, keys: str|bytes|memoryview|Iterable, on: int=0,
vals: str|bytes|memoryview|Iterable = b''):
"""
Pins (sets) vals at key made from keys with on suffix in insertion order
using hidden ordinal proem. Overwrites. Removes all pre-existing vals
that share same keys and replaces them with vals
Parameters:
keys (Iterable): of key strs to be combined in order to form key
on (int): ordinal number used with onKey(pre,on) to form onkey.
vals (Iterable): str serializations
Returns:
result (bool): True If successful, False otherwise.
"""
key = self._tokey(keys)
self.db.delOnIoDups(db=self.sdb,
key=key,
on=on,
sep=self.sep.encode())
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) if vals else () # make iterable
return self.db.putOnIoDupVals(db=self.sdb,
key=key,
on=on,
vals=tuple(self._ser(val) for val in vals),
sep=self.sep.encode())
[docs]
def add(self, keys: str | bytes | memoryview | Iterable, on: int=0,
val: str | bytes | memoryview = ''):
"""Add val idempotently at key made from keys with on suffix
in insertion order using hidden ordinal proem. Idempotently means do not
add any val that is already in dup vals at key. Does not overwrite.
Parameters:
keys (str | bytes | memoryview | Iterable): top keys as prefix to be
combined with serialized on suffix and sep to form onkey
on (int): ordinal number used with onKey(pre,on) to form onkey.
val (str | bytes | memoryview): serialization
Returns:
result (bool): True means unique value added among duplications,
False means duplicate of same value already exists.
"""
return (self.db.addOnIoDupVal(db=self.sdb,
key=self._tokey(keys),
on=on,
val=self._ser(val),
sep=self.sep.encode()))
[docs]
def append(self, keys: str | bytes | memoryview,
val: str | bytes | memoryview):
"""
Returns:
on (int): ordinal number of newly appended val
Parameters:
keys (str | bytes | memoryview | Iterable): top keys as prefix to be
combined with serialized on suffix and sep to form key
val (str | bytes | memoryview): serialization
"""
return (self.db.appendOnIoDupVal(db=self.sdb,
key=self._tokey(keys),
val=self._ser(val),
sep=self.sep.encode()))
[docs]
def get(self, keys: str|bytes|memoryview|Iterable, on: int = 0):
"""Gets dup vals list at key made from keys
Parameters:
keys (str|bytes|memoryview|Iterable): of key strs to be
combined in order to form key
on (int): ordinal number used with onKey(pre,on) to form key.
Returns:
vals (list[str]): values if any else empty tuuple
"""
return [self._des(val) for val in
self.db.getOnIoDupVals(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())]
[docs]
def getItemIter(self, keys: str|bytes|memoryview|Iterable, on: int=0, ion: int=0):
"""Iterates over dup items (key, on, val) at onkey made from keys and on
in insertion order from offset ion >= ion into set using hidden
ordinal proem.
When effective onkey is empty or missing then returns empty iterator
Returns:
items (Iterator[str]): deserialized item elements of dups at onkey
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form key.
ion (int): starting insertion ordinal value, default 0
"""
for key, on, val in self.db.getOnIoDupItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode()):
yield (self._tokeys(key), on, self._des(val))
[docs]
def getIter(self, keys: str|bytes|memoryview|Iterable, on: int=0, ion: int=0):
"""Iterates over dup vals at onkey made from keys and on
in insertion order from offset ion >= ion into set using hidden ordinal proem.
When effective onkey is empty or missing then returns empty iterator
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form key.
ion (int): starting insertion ordinal value, default 0
Returns:
vals (Iterator[bytes]): deserialized val of dups at onkey
"""
for key, on, val in self.db.getOnIoDupItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode()):
yield (self._des(val))
[docs]
def getLast(self, keys: str|bytes|memoryview|Iterable, on: int = 0):
"""Gets last val inserted at key made from keys in insertion order using
hidden ordinal proem.
Parameters:
keys (Iterable): of key strs to be combined in order to form key
on (int): ordinal number used with onKey(pre,on) to form key.
Returns:
last (str): value str, None if no entry at effective key made from
keys and on
"""
val = self.db.getOnIoDupLast(db=self.sdb, key=self._tokey(keys), on=on)
return (self._des(val) if val is not None else val)
[docs]
def rem(self, keys: str | bytes | memoryview | Iterable, on: int=0,
val: str|bytes|memoryview|None = None):
"""Removes entry at key made from keys and dup val that matches val if any,
notwithstanding hidden ordinal proem. Otherwise deletes all dup values
at key if any.
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized on suffix and sep to form onkey
on (int): ordinal number used with onKey(pre,on) to form key.
val (str|bytes|memoryview|None): value at key to delete. Subclass ._ser method may
accept different value types
if val is None then remove all values at key
Returns:
result (bool): True if onkey with dup val exists so rem successful.
False otherwise
"""
if val is not None:
return self.db.delOnIoDupVal(db=self.sdb,
key=self._tokey(keys),
on=on,
val=self._ser(val),
sep=self.sep.encode())
else:
return self.db.delOnIoDups(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())
[docs]
def cnt(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""Counts iodup values at onkey made from keys and on.
When keys is empty then counts whole db.
Return count of dup values at key made from keys with hidden ordinal
proem. Zero otherwise
Parameters:
keys (str | bytes | memoryview | Iterable): of key parts to be
combined in order to form key
on (int): ordinal number used with onKey(pre,on) to form key.
"""
if not keys:
return self.db.cntAll(db=self.sdb)
return (self.db.cntOnIoDups(db=self.sdb, key=self._tokey(keys),
on=on, sep=self.sep.encode()))
[docs]
def getTopItemIter(self, keys: str|bytes|memoryview|Iterable=""):
"""Iterates over top branch of all insertion ordered dup values where
each key startwith top.
Assumes every effective key in db has trailing on element,
onkey = key + sep + on, so can return on in item.
Also assumes every effective key includes hiddion isertion ordinal ion
suffix that is suffixed and unsuffixed transparently.
When top key is empty, gets all items in database.
Returns:
items (Iterator[(tuple, int, str)]): iterator of triples
(keys, on, val)
where keys forms base key, on is int, and val is entry value at
with insertion ordering suffix removed from effective key.
Parameters:
keys (str|bytes|memoryview|Iterable): keys as truncated top key,
to get a key space prefix to get all the items
from multiple branches of the key space.
If top key is empty then gets all items in database.
on (int): ordinal number used with onKey(pre,on) to form key.
"""
for keys, on, val in (self.db.getOnTopIoDupItemIter(db=self.sdb,
top=self._tokey(keys),
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""Iterates over all items of each dup set for all on >= on for key.
When on == 0, default, Iterates over alls items of each set for all on for key.
When key is empty then iterates over all items in whole db
Items are triples of (keys, on, val)
Returns:
items (Iterator[(top keys, on, val)]): triples of (onkeys, on int,
deserialized val)
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized on suffix and sep to form onkey
When keys is empty then retrieves whole database including duplicates
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnIoDupItemIterAll(db=self.sdb,
key=self._tokey(keys), on=on, sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""Iterates over all values of each dup set for all on >= on for key.
When on == 0, default, Iterates over alls values of each dup set for
all on for key.
When key is empty then iterates over all items in whole db
Returns
vals (Iterator[bytes]): iterator of dup vals at each onkey but
increments of on >= on i.e. all key.on beginning with on
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized on suffix and sep to form onkey
When keys is empty then retrieves whole database including duplicates
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for val in (self.db.getOnIoDupIterAll(db=self.sdb,
key=self._tokey(keys), on=on, sep=self.sep.encode())):
yield (self._des(val))
[docs]
def getLastIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""
Returns
last (Iterator[bytes]): deserialized last duplicate val of of each
onkey
Parameters:
keys (str | bytes | memoryview | iterator): top keys as prefix to be
combined with serialized on suffix and sep to form key
When keys is empty then retrieves whole database including duplicates
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for val in (self.db.getOnIoDupLastValIter(db=self.sdb,
key=self._tokey(keys), on=on, sep=self.sep.encode())):
yield (self._des(val))
[docs]
def getLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""
Returns
items (Iterator[(top keys, on, val)]): triples of (keys, on int,
deserialized val) last duplicate item as each onkey where onkey
is the key+serialized on
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized on suffix and sep to form key
When keys is empty then retrieves whole database including duplicates
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnIoDupLastItemIter(db=self.sdb,
key=self._tokey(keys), on=on, sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getItemBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""
Returns:
items (Iterator[(top keys, on, val)]): triples of (onkeys, on int,
deserialized val) in reverse order
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized on suffix and sep to form onkey
When keys is empty then retrieves whole database including duplicates
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnIoDupItemBackIter(db=self.sdb,
key=self._tokey(keys), on=on, sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0):
"""
Returns
val (Iterator[bytes]): deserialized val of of each
onkey in reverse order
Parameters:
keys (str | bytes | memoryview | iterator): keys as prefix to be
combined with serialized on suffix and sep to form onkey
When keys is empty then retrieves whole database including duplicates
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for val in (self.db.getOnIoDupValBackIter(db=self.sdb,
key=self._tokey(keys), on=on, sep=self.sep.encode())):
yield (self._des(val))
[docs]
class B64OnIoDupSuber(B64SuberBase, OnIoDupSuber):
"""
Subclass of B64SuberBase and OnIoDupSuber that serializes and deserializes
values as .sep joined strings of Base64 components in insertion ordered
duplicates with leading value proem but where the trailing part of the
key space is a serialized monotonically increasing ordinal number.
Proem + .sep joined value and must fit n 511 bytes of keyspace as duplicate.
Insertion order is maintained by automagically prepending and stripping an
ordinal ordering proem to/from each duplicate value at a given key.
OnIoDupSuber adds the convenience methods from OnSuberBase to OnIoDupSuber
for those cases where the keyspace has a trailing ordinal part.
Assumes dupsort==True
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to True
sep (str): separator for combining keys tuple of strs into key bytes
for db key and also used to convert val iterator to val bytes
Must not be Base64 character.
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(B64OnIoDupSuber, self).__init__(*pa, **kwa)
[docs]
class OnIoSetSuber(OnSuberBase, IoSetSuber):
"""Sub class of IoSetSuber and OnSuberBase that supports Insertion Ordered
Set values (IoSet) with an exposed ordinal tail in each effective key.
To clarify, there are two ordinals, one exposed as the tail end of the
effective key space followed by a hidden suffix for the set value space.
OnIoSetSuber adds the convenience methods from OnSuberBase to IoSetSuber for
those cases where the keyspace has a trailing ordinal part.
Insertion order is maintained by automagically prepending
and stripping an ordinal ordering suffix to/from each unique value at a given
effective key.
An IoSet is a set of distinct entries at a given effective
database key but with dupsort==False. Effective key means that there are
multiple values in a set of values where every member of the set has the
same key (duplicate key) but distinct values (a set not a list).
The set of values is an ordered set using insertion order. Any given value
may appear only once in the set (not a list).
This works similarly to the IO value duplicates for the LMDBer class with a
sub db of LMDB (dupsort==True) but without its size limitation of 511 bytes.
Here the key is augmented with a hidden numbered suffix that provides a
an insertion ordered set of values at each effective key (duplicate key).
The suffix is appended and stripped transparently. The set of multiple items with
duplicate keys are retrieved in insertion order when iterating or as a list
of the set elements.
Combined with an On, ordinal numbered key space means that there are two ordinals
at the tail (right) end of each key. The rightmost is the insertion ordering
suffix ordinal. The next to the left is the exposed ordinal number tail used for
external ordering. The insertion ordinal suffix is hidden. It is transparently added
and stripped. To clarify, the exposed tail part of the key space is
a serialized monotonically increasing ordinal number. Following that is a
hidden insertion ordering ordinal suffix that defines the order within the set.
Each memeber of a set has the same effective key including the exposed ordinal tail
but a different hidden suffix.
This is useful for escrows of key events which are ordinally numbered such
as sn but where likely events with different overall value but same sn must be
maintained in insertion order.
OnIoSetSuber may be less performant then IoDupSuber for values that are indices
to other sub dbs that fit the size constraint because LMDB support for
duplicates is more space efficient and code performant but has a 511 bytes
maximum.
Duplicates at a given key preserve insertion order of duplicate.
Because lmdb is lexocographic an insertion ordering proem is prepended to
all values that makes lexocographic order that same as insertion order.
Duplicates are ordered as a pair of key plus value so prepending proem
to each value changes duplicate ordering. Proem is 33 characters long.
With 32 character hex string followed by '.' for essentiall unlimited
number of values which will be limited by memory.
To insure set semantics (i.e. no duplicate values in set) using a suffix
ordinal insertions must explicity check for duplicate set values
before insertion. A python set is used for the inclusion test.
Set inclusion scales with O(1) whereas list inclusion scales with O(n).
"""
[docs]
def __init__(self, *pa, **kwa):
"""Initialize instance
Inherited Parameters:
"""
super(OnIoSetSuber, self).__init__(*pa, **kwa)
[docs]
def put(self, keys: str|bytes|memoryview|Iterable,
on: int=0,
vals: str|bytes|memoryview|Iterable|None=None):
"""Put all vals idempotently at key at key made from keys with exposed
on tail in insertion order using hidden ordinal suffix. Idempotently
means do not put any val in vals that is already in set vals at key.
Does not overwrite.
Returns:
result (bool): True If successful, False otherwise.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form onkey.
vals (str|bytes|memoryview|NonStrIterable|None): serialized values
to add to set of vals at onkey if any.
When not NonStrIterable converts to iterable.
Empty iterable or None returns False
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) if vals else () # make iterable
return self.db.putOnIoSetVals(db=self.sdb,
key=self._tokey(keys),
on=on,
vals=tuple(self._ser(val) for val in vals),
sep=self.sep.encode())
[docs]
def pin(self, keys: str|bytes|memoryview|Iterable,
on: int=0,
vals: str|bytes|memoryview|Iterable|None=None):
"""Pins (sets) vals at key made from keys with exposed
on tail in insertion order using hidden ordinal suffix. Overwrites.
Removes all pre-existing vals that share same onkey and replaces
them with vals
Returns:
result (bool): True If successful, False otherwise.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form onkey.
vals (NonStrIterable|None): serialized values to replace set of vals at
onkey if any. None means empty iterable.
When not NonStrIterable converts to iterable.
Empty iterable returns False
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) if vals else () # make iterable
return self.db.pinOnIoSetVals(db=self.sdb,
key=self._tokey(keys),
on=on,
vals=tuple(self._ser(val) for val in vals),
sep=self.sep.encode())
[docs]
def append(self, keys: str|bytes|memoryview|Iterable,
vals: str|bytes|memoryview|Iterable|None=None):
"""Appends vals to next highest unused exposed ordinal tail and returns the
ordinal. If vals None or empty raises ValueError.
Returns:
on (int): ordinal number tail of newly appended val
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
vals (NonStrIterable|None): values to append
None means empty iterable.
Empty iterable raises ValueError
"""
if not helping.isNonStringIterable(vals): # not iterable
vals = (vals, ) if vals else () # make iterable
return (self.db.appendOnIoSetVals(db=self.sdb,
key=self._tokey(keys),
vals=tuple(self._ser(val) for val in vals),
sep=self.sep.encode()))
[docs]
def add(self, keys: str|bytes|memoryview|Iterable,
on: int=0,
val: str|bytes|memoryview |None=None):
"""Add val idempotently at key made from keys with exposed on tail in
insertion order using hidden ordinal suffix. Idempotently means do not
add any val that is already in set vals at effective key. Does not overwrite.
When val None returns False
Returns:
result (bool): True means unique value added to set,
False means value already in set.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form onkey.
val (str|bytes|memoryview|None): value to add.
when None returns False
"""
return (self.db.addOnIoSetVal(db=self.sdb,
key=self._tokey(keys),
on=on,
val=self._ser(val),
sep=self.sep.encode()))
[docs]
def getItem(self, keys: str|bytes|memoryview|Iterable, on: int=0, ion: int=0):
"""Gets list of items (key, on, val) from set of entries at onkey
made from keys and on starting at offset ion into set.
When onkey missing or key empty or missing returns empty list.
Returns
item (tuple[bytes, int, str|bytes|memoryview]|None): at onkey if any
of form (key, on, val)
None if no entry at onkey or key empty or None
Parameters:
keys (str|bytes|memoryview|Iterable): keys as prefix to be
combined with serialized exposed on tail and sep to form onkey
on (int): ordinal number used with onKey(key ,on) to form key.
"""
return [(self._tokeys(key), on, self._des(val)) for key, on, val in
self.db.getOnIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode())]
[docs]
def get(self, keys: str|bytes|memoryview|Iterable, on: int=0, ion: int=0):
"""Gets set vals list at onkey made from keys and on in insertion order from
from offset ion into set using hidden ordinal suffix.
When onkey is empty or missing then returns empty list
Returns:
vals (list[str]): values if any else empty tuuple
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
When key is empty then returns empty iterator
on (int): ordinal number tail used with onKey(pre,on) to form key.
ion (int): starting insertion ordinal value, default 0
"""
return [self._des(val) for key, on, val in
self.db.getOnIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode())]
[docs]
def getItemIter(self, keys: str|bytes|memoryview|Iterable, on: int=0, ion: int=0):
"""Iterates over set items (key, on, val) at onkey made from keys and on
in insertion order from from offset ion into set using hidden ordinal suffix.
When onkey is empty or missing then returns empty iterator
Returns:
items (Iterator[str]): deserialized items elements of set at onkey
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form key.
ion (int): starting insertion ordinal value, default 0
"""
for key, on, val in self.db.getOnIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode()):
yield (self._tokeys(key), on, self._des(val))
[docs]
def getIter(self, keys: str|bytes|memoryview|Iterable, on: int=0, ion: int=0):
"""Iterates over set vals at onkey made from keys and on in insertion order
from from offset ion into set using hidden ordinal suffix.
When onkey is empty or missing then returns empty iterator
Returns:
val (Iterator[str]): deserialized val elements of set at onkey
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number tail used with onKey(pre,on) to form key.
ion (int): starting insertion ordinal value, default 0
"""
for key, on, val in self.db.getOnIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode()):
yield (self._des(val))
[docs]
def getLastItem(self, keys: str|bytes|memoryview|Iterable, on: int=0):
"""Gets last item inserted at key made from keys in insertion order using
hidden ordinal proem.
Returns:
last (tuple[tuple, int, str]): last set item triple at onkey
(keys, on, val)
Empty tuple () if onkey not in db or key empty.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number used with onKey(pre,on) to form key.
"""
if last := self.db.getOnIoSetLastItem(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode()):
key, on, val = last
return (self._tokeys(key), on, self._des(val))
return ()
[docs]
def getLast(self, keys: str|bytes|memoryview|Iterable, on: int=0):
"""Gets last val inserted at key made from keys in insertion order using
hidden ordinal proem.
Returns:
last (str|None): value str, None if no entry at effective key made from
keys and on
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number used with onKey(pre,on) to form key.
"""
if last := self.db.getOnIoSetLastItem(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode()):
key, on, val = last
return self._des(val)
return None
[docs]
def rem(self, keys: str|bytes|memoryview|Iterable,
on: int=0,
val: str|bytes|memoryview|None=None):
"""Removes entry in set with val if any at key made and on in insertion order
using hidden ordinal suffix.
When val matches a value in the set at the effective key
then only that value is removed.
When val is None then removes all values from set effectively deleting
entry at onkey = keys + sep + on if any in db.
Returns:
result (bool): True if onkey with dup val exists so rem successful.
False otherwise
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number used with onKey(pre,on) to form key.
val (str|bytes|memoryview|None): value at key to remove.
Subclass ._ser method may accept different value types
if val is None then remove all values at key
"""
return self.db.remOnIoSetVal(db=self.sdb,
key=self._tokey(keys),
on=on,
val=self._ser(val) if val is not None else val,
sep=self.sep.encode())
[docs]
def remAll(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Removes all entries for all sets for all on >= on at key.
When on i==0, default, then removes all entries for all sets for all on at key.
When key is empty then removes whole db.
Returns:
result (bool): True if onkey with dup val exists so rem successful.
False otherwise
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
When key is empty then remove all entries in whole db
on (int): base key. None means all on for key
"""
return self.db.remOnAllIoSet(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())
[docs]
def cnt(self, keys: str|bytes|memoryview|Iterable="", on: int=0, ion: int=0):
"""Counts all entries in set at onkey = keys + sep + on starting at
offset suffix ion into set.
Returns:
count (int): count values in set at effective onkey from insertion
ordering offset ion.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key.
When empty counts whole db.
on (int): ordinal number used with onKey(pre,on) to form key.
ion (int): starting insertion ordinal offset into set, default 0.
"""
if not keys:
return self.db.cntOnAllIoSet(db=self.sdb)
return (self.db.cntOnIoSet(db=self.sdb,
key=self._tokey(keys),
on=on,
ion=ion,
sep=self.sep.encode()))
[docs]
def cntAll(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Counts all set entries for all on >= on at key.
When on == 0, default, then count all set members for all on for key
When key is empty then count all on for all key i.e. whole db
Returns:
count (int): count of set members for onkey for on >= on. When on is
None then count of all on for key. When key is empty
then count of all on for all key for whole db.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int): ordinal number used with onKey(pre,on) to form key.
"""
return (self.db.cntOnAllIoSet(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode()))
[docs]
def getTopItemIter(self, keys: str|bytes|memoryview|Iterable=""):
"""Iterates over top branch of all insertion ordered set values where
each key startwith top.
Assumes every effective key in db has trailing on element,
onkey = key + sep + on, so can return on in item.
Also assumes every effective key includes hiddion isertion ordinal ion
suffix that is suffixed and unsuffixed transparently.
When top key is empty, gets all items in database.
Returns:
items (Iterator[(tuple, int, str)]): iterator of triples
(keys, on, val)
where keys forms base key, on is int, and val is entry value at
with insertion ordering suffix removed from effective key.
Parameters:
keys (str|bytes|memoryview|Iterable): keys as truncated top key,
to get a key space prefix to get all the items
from multiple branches of the key space.
If top key is empty then gets all items in database.
on (int): ordinal number used with onKey(pre,on) to form key.
"""
for keys, on, val in (self.db.getOnTopIoSetItemIter(db=self.sdb,
top=self._tokey(keys),
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllItemIter(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Iterates over all items of each set for all on >= on for key.
When on ==0, default Iterates over alls items of each set for all on for key.
When key is empty then iterates over all items in whole db
Items are triples of (keys, on, val)
Returns:
items (Iterator[(tuple, int, str)]): iterator of triples
(keys, on, val)
where keys forms base key, on is int, and val is entry value at
with insertion ordering suffix removed from effective key.
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
from which to make onkey = key + sep + on
If keys is empty then gets all items in database.
on (int): ordinal number used with onKey(pre,on) to form key.
When on is None then iterates all on at key,
"""
for keys, on, val in (self.db.getOnAllIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllIter(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Iterates over alls values of each set for all on >= on for key.
When on ==0, default, Iterates over alls items of each set for all on for key.
When key is empty then iterates over all items in whole db
Items are triples of (keys, on, val)
Returns
val (Iterator[str]): deserialized val of each matching entry
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
When keys is empty then retrieves whole database including all
set values at each effective key.
on (int): ordinal number used with onKey(pre,on) to form key.
When on ==0 then iterates over all items of all sets for all on
for all key >= key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._des(val))
[docs]
def getAllLastItemIter(self, keys: str|bytes|memoryview|Iterable="",
on: int=0):
"""Iterates over last items of each set for all on >= on at key.
When on is None iterates over last items of each set for all on at key.
When key is empty iterates over last items of all sets at all keys in db.
Returns
last (Iterator[(tuple, int, str)]): last set item triples of
(keys, on, val) where onkey = key + sep + on
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
When keys is empty then retrieves whole database including all
set items
on (int|None): ordinal number used with onKey(pre,on) to form key.
When None then all on for all key >= key
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetLastItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllLastIter(self, keys: str|bytes|memoryview|Iterable="", on: int=0):
"""Iterates over last value of each set for all on >= on at key
When on ==0, default, iterates over last value of each set for all on at key
When key is empty iterates over last value of each set of all on for
all keys, i.e. whole db.
Returns
last (Iterator[str]): deserialized last set val of of each onkey
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
When keys is empty then retrieves whole database including all
set values
on (int): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetLastItemIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._des(val))
[docs]
def getAllItemBackIter(self, keys: str|bytes|memoryview|Iterable="",
on: int|None=None):
"""Iterates backwards over all set items for all on <= on at key.
When on is None iterates backwards over all set items for all on at key.
When key empty then iterates backwards over whole db.
Returned items are triples of (key, on, val)
Raises StopIterationError when done or when key empty or None
Backwards means decreasing numerical value of ion, for each on and
decreasing numerical value on for each key and decreasing lexocographic
order of each key.
Returns:
item (Iterator[(tuple, int, str)]): triples of (keys, on, val)
in backwards order
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
keys empty means whole db
on (int|None): ordinal number used with onKey(pre,on) to form key.
None means iterate over all on for key
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetItemBackIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllBackIter(self, keys: str|bytes|memoryview|Iterable = "",
on: int|None=None):
"""Iterates backwards over all set values for all on <= on at key.
When on is None iterates backwards over all set values for all on at key.
When key empty then iterates backwards over whole db.
Backwards means decreasing numerical value of ion, for each on and
decreasing numerical value on for each key and decreasing lexocographic
order of each key.
Raises StopIterationError when done or when key empty or None
Returns
val (Iterator[str]): deserialized vals in backwards order
Parameters:
keys (str | bytes | memoryview | Iterable): key(s) made into base key
on (int|None): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetItemBackIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._des(val))
[docs]
def getAllLastItemBackIter(self, keys: str|bytes|memoryview|Iterable="",
on: int|None=None):
"""Iterates backwards over last set items for all on <= on at key.
When on is None iterates backwards over last set items for all on at key.
When key empty then iterates backwards over whole db for last set items.
Returned items are triples of (key, on, val)
Raises StopIterationError when done or when key empty or None
Backwards means decreasing numerical value of ion, for each on and
decreasing numerical value on for each key and decreasing lexocographic
order of each key.
Returns:
lastitem (Iterator[(tuple, int, str)]): triples of (keys, on, val)
in backwards order
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
keys empty means whole db
on (int|None): ordinal number used with onKey(pre,on) to form key.
None means iterate over all on for key
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetLastItemBackIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._tokeys(keys), on, self._des(val))
[docs]
def getAllLastBackIter(self, keys: str|bytes|memoryview|Iterable = "",
on: int|None=None):
"""Iterates backwards over last set values for all on <= on at key.
When on is None iterates backwards over last set values for all on at key.
When key empty then iterates backwards over whole db for last set values.
Backwards means decreasing numerical value of ion, for each on and
decreasing numerical value on for each key and decreasing lexocographic
order of each key.
Raises StopIterationError when done or when key empty or None
Returns
lastval (Iterator[str]): deserialized vals in backwards order
Parameters:
keys (str|bytes|memoryview|Iterable): key(s) made into base key
on (int|None): ordinal number used with onKey(pre,on) to form key.
sep (bytes): separator character for split
"""
for keys, on, val in (self.db.getOnAllIoSetLastItemBackIter(db=self.sdb,
key=self._tokey(keys),
on=on,
sep=self.sep.encode())):
yield (self._des(val))
[docs]
class B64OnIoSetSuber(B64SuberBase, OnIoSetSuber):
"""Subclass of B64SuberBase and OnIoSetSuber that serializes and deserializes
values as .sep joined strings of Base64 components in insertion order using
hidden ion suffix in keyspace. Using IoSet removes 511 byte limitation of
duplicates (dupsort=True). The last keyspece element before the hidden
insertion ordering suffix is an ordinal (hence On). This allows entries
at a key prefix to be stored monotonically as per sequence numbering.
Insertion order within each set is maintained by automagically suffixing
and unsuffixing the insertion ordering suffix.
OnIoSetSuber adds the convenience methods from OnSuberBase to OnIoSetSuber
for those cases where the keyspace has a trailing ordinal part.
Assumes dupsort==False
"""
[docs]
def __init__(self, *pa, **kwa):
"""
Inherited Parameters:
db (LMDBer): base db
subkey (str): LMDB sub database key
dupsort (bool): True means enable duplicates at each key
False (default) means do not enable duplicates at
each key. Set to False
sep (str): separator for combining keys tuple of strs into key bytes
for db key and also used to convert val iterator to val bytes
Must not be Base64 character.
default is self.Sep == '.'
verify (bool): True means reverify when ._des from db when applicable
False means do not reverify. Default False
"""
super(B64OnIoSetSuber, self).__init__(*pa, **kwa)