Date HWM#

class etl_entities.hwm.date_hwm.DateHWM(*, source: <pydantic.fields.DeferredType object at 0x7f7824aeb9d0>, value: ~datetime.date | None = None, modified_time: <pydantic.fields.DeferredType object at 0x7f7824aebb50> = None, process: <pydantic.fields.DeferredType object at 0x7f7824aebc10> = None, column: <pydantic.fields.DeferredType object at 0x7f7824aebcd0>)#

Date HWM type

Parameters:
columnetl_entities.source.db.column.Column

Column instance

sourceetl_entities.source.db.table.Table

Table instance

valuedatetime.date or None, default: None

HWM value

modified_timedatetime.datetime, default: current datetime

HWM value modification time

processetl_entities.process.process.Process, default: current process

Process instance

Examples

from datetime import date
from etl_entities import DateHWM, Column, Table

column = Column(name="id")
table = Table(name="mytable", db="mydb", instance="postgres://db.host:5432")

hwm = DateHWM(column=column, source=table, value=date(year=2021, month=12, day=3))
Attributes:
name

HWM column name

qualified_name

Unique name of HWM

Methods

copy(*[, include, exclude, update, deep])

Duplicate a model, optionally choose which fields to include, exclude and change.

covers(value)

Return True if input value is already covered by HWM

deserialize(inp)

Return HWM from dict representation

dict(*[, include, exclude, by_alias, ...])

Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.

json(*[, include, exclude, by_alias, ...])

Generate a JSON representation of the model, include and exclude arguments as per dict().

serialize()

Return dict representation of HWM

set_value(value)

Replaces current HWM value with the passed one, and return HWM.

update(value)

Updates current HWM value with some implementation-specific logic, and return HWM.
__add__(value)#

Increase HWM value and return copy of HWM

Returns:
resultColumnHWM

HWM copy with new value

Examples

# assume val2 == val1 + inc

hwm1 = ColumnHWM(value=val1, ...)
hwm2 = ColumnHWM(value=val2, ...)

# same as ColumnHWM(value=hwm1.value + inc, ...)
assert hwm1 + inc == hwm2
__bool__()#

Check if HWM value is set

Returns:
resultbool

False if value is None, True otherwise

Examples

from etl_entities import ColumnHWM

hwm = ColumnHWM(value=1, ...)
assert hwm  # same as hwm.value is not None

hwm = ColumnHWM(value=None, ...)
assert not hwm
__eq__(other)#

Checks equality of two HWM instances

Returns:
resultbool

True if both inputs are the same, False otherwise.

Examples

from datetime import date
from etl_entities import DateHWM

hwm1 = DateHWM(value=date(year=2021, month=12, day=30), ...)
hwm2 = DateHWM(value=date(year=2021, month=12, day=31), ...)

assert hwm1 == hwm1
assert hwm1 != hwm2
__lt__(other)#

Checks current HWM value is less than another one

Returns:
resultbool

True if current HWM value is less than provided value, False otherwise.

Examples

from datetime import date
from etl_entities import DateHWM

hwm1 = DateHWM(value=date(year=2021, month=12, day=30), ...)
hwm2 = DateHWM(value=date(year=2021, month=12, day=31), ...)

assert hwm1 < hwm2
assert hwm2 > hwm1

assert hwm1 < date(year=2021, month=12, day=1)
assert hwm1 > date(year=2021, month=12, day=31)

hwm3 = DateHWM(value=None, ...)
assert hwm1 < hwm3  # will raise TypeError
assert hwm1 < None  # same thing
__sub__(value)#

Decrease HWM value, and return copy of HWM

Returns:
resultColumnHWM

HWM copy with new value

Examples

# assume val2 == val1 - dec

hwm1 = ColumnHWM(value=val1, ...)
hwm2 = ColumnHWM(value=val2, ...)

# same as ColumnHWM(value=hwm1.value - dec, ...)
assert hwm1 - dec == hwm2
copy(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, update: DictStrAny | None = None, deep: bool = False) → Model#

Duplicate a model, optionally choose which fields to include, exclude and change.

Parameters:

  • include – fields to include in new model

  • exclude – fields to exclude from new model, as with values this takes precedence over include

  • update – values to change/add in the new model. Note: the data is not validated before creating the new model: you should trust this data

  • deep – set to True to make a deep copy of the model

Returns:

new model instance

covers(value: ColumnValueType) → bool#

Return True if input value is already covered by HWM

Examples

column = Column(name="id")
table = Table(name="mytable", db="mydb", instance="postgres://db.host:5432")

hwm = ColumnHWM(column=column, source=table, value=1)

assert hwm.covers(0)  # 0 <= 1
assert hwm.covers(1)  # 1 <= 1
assert hwm.covers(0.5)  # 0.5 <= 1
assert not hwm.covers(2)  # 2 > 1

empty_hwm = ColumnHWM(column=column, source=table)

assert not empty_hwm.covers(0)  # non comparable with None
assert not empty_hwm.covers(1)  # non comparable with None
assert not empty_hwm.covers(0.5)  # non comparable with None
assert not empty_hwm.covers(2)  # non comparable with None
classmethod deserialize(inp: dict)#

Return HWM from dict representation

Returns:
resultHWM

Deserialized HWM

Examples

from etl_entities import IntHWM

assert IntHWM.deserialize(
    {
        "value": "1",
        "type": "int",
        "column": {"name": ..., "partition": ...},
        "source": ...,
        "process": ...,
    }
) == IntHWM(value=1, ...)

IntHWM.deserialize({"type": "date"})  # raises ValueError
dict(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) → DictStrAny#

Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.

json(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Callable[[Any], Any] | None = None, models_as_dict: bool = True, **dumps_kwargs: Any) → unicode#

Generate a JSON representation of the model, include and exclude arguments as per dict().

encoder is an optional function to supply as default to json.dumps(), other arguments as per json.dumps().

property name: str#

HWM column name

Returns:
valuestr

Column name

Examples

column = Column(name="id")
table = Table(name="mytable", db="mydb", instance="postgres://db.host:5432")

hwm = ColumnHWM(column=column, source=table, value=val)

assert hwm.name == "id"
property qualified_name: str#

Unique name of HWM

Returns:
valuestr

Qualified name

Examples

column = Column(name="id")
table = Table(name="mytable", db="mydb", instance="postgres://db.host:5432")

hwm = ColumnHWM(column=column, source=table, value=1)

assert (
    hwm.qualified_name
    == "id#mydb.mytable@postgres://db.host:5432#currentprocess@currenthost"
)
serialize() → dict#

Return dict representation of HWM

Returns:
resultdict

Serialized HWM

Examples

from etl_entities import IntHWM

hwm = IntHWM(value=1, ...)
assert hwm.serialize() == {
    "value": "1",
    "type": "int",
    "column": {"name": ..., "partition": ...},
    "source": ...,
    "process": ...,
}
set_value(value: ValueType) → HWM#

Replaces current HWM value with the passed one, and return HWM.

Note

Changes HWM value in place instead of returning new one

Returns:
resultHWM

Self

Examples

from etl_entities import IntHWM

hwm = IntHWM(value=1, ...)

hwm.set_value(2)
assert hwm.value == 2
update(value: ColumnValueType)#

Updates current HWM value with some implementation-specific logic, and return HWM.

Note

Changes HWM value in place

Returns:
resultColumnHWM

HWM copy with new value

Examples

from etl_entities import IntHWM

hwm = IntHWM(value=1, ...)

hwm.update(2)
assert hwm.value == 2

hwm.update(1)
assert hwm.value == 2  # value cannot decrease