Integer HWM#

class etl_entities.old_hwm.int_hwm.IntHWM(*, source: <pydantic.v1.fields.DeferredType object at 0x7ffb11e257f0>, value: ~pydantic.v1.types.StrictInt | None = None, modified_time: <pydantic.v1.fields.DeferredType object at 0x7ffb11e25970> = None, process: <pydantic.v1.fields.DeferredType object at 0x7ffb11e25a30> = None, column: <pydantic.v1.fields.DeferredType object at 0x7ffb11e25af0>)#

Integer HWM type

Deprecated since version 2.0.0: Use ColumnIntHWM instead

Parameters:

columnetl_entities.source.db.column.Column: Column instance
sourceetl_entities.source.db.table.Table: Table instance
valueint or None, default: None: HWM value
modified_timedatetime.datetime, default: current datetime: HWM value modification time
processetl_entities.process.process.Process, default: current process: Process instance

Examples

from etl_entities.old_hwm import IntHWM
from etl_entities.source import Column, Table

column = Column(name="id")
table = Table(name="mydb.mytable", instance="postgres://db.host:5432")

old_hwm = IntHWM(column=column, source=table, value=1)

__add__(value)#

Increase HWM value and return copy of HWM

Parameters:

valueAny or None

Should be compatible with value attribute type.

For example, you cannot add str to int value, but you can add int to int.

Returns:

resultColumnHWM: HWM copy with new value

Examples

# assume val2 == val1 + inc

hwm1 = ColumnHWM(value=val1, ...)
hwm2 = ColumnHWM(value=val2, ...)

# same as ColumnHWM(value=hwm1.value + inc, ...)
assert hwm1 + inc == hwm2

__bool__()#

Check if HWM value is set

Returns:

resultbool: False if value is None, True otherwise

Examples

from etl_entities.old_hwm import ColumnHWM

old_hwm = ColumnHWM(value=1, ...)
assert old_hwm  # same as old_hwm.value is not None

old_hwm = ColumnHWM(value=None, ...)
assert not old_hwm

__eq__(other)#

Checks equality of two HWM instances

Parameters:

otheretl_entities.old_hwm.int_hwm.IntHWM or int

Should be comparable with value attribute type.

You can compare two int values, but you cannot compare int with date value, as well as different HWM types, like etl_entities.old_hwm.int_hwm.IntHWM and etl_entities.old_hwm.date_hwm.DateHWM.

Returns:

resultbool: True if both inputs are the same, False otherwise.

Examples

from etl_entities.old_hwm import IntHWM

hwm1 = IntHWM(value=1, ...)
hwm2 = IntHWM(value=2, ...)

assert hwm1 == hwm1
assert hwm1 != hwm2

__lt__(other)#

Checks current HWM value is less than another one

Parameters:

otheretl_entities.old_hwm.int_hwm.IntHWM or int

Should be comparable with value attribute type.

Warning

You cannot compare HWMs if one of them has None value

Returns:

resultbool: True if current HWM value is less than provided value, False otherwise.

Examples

from etl_entities.old_hwm import IntHWM

hwm1 = IntHWM(value=1, ...)
hwm2 = IntHWM(value=2, ...)

assert hwm1 < hwm2
assert hwm1 > hwm2

assert hwm1 < 2
assert hwm1 > 0

hwm3 = IntHWM(value=None, ...)
assert hwm1 < hwm3  # will raise TypeError
assert hwm1 < None  # same thing

__sub__(value)#

Decrease HWM value, and return copy of HWM

Parameters:

valueAny or None

Should be compatible with value attribute type.

For example, you cannot subtract str from int value, but you can subtract int from int.

Returns:

resultColumnHWM: HWM copy with new value

Examples

# assume val2 == val1 - dec

hwm1 = ColumnHWM(value=val1, ...)
hwm2 = ColumnHWM(value=val2, ...)

# same as ColumnHWM(value=hwm1.value - dec, ...)
assert hwm1 - dec == hwm2

Duplicate a model, optionally choose which fields to include, exclude and change.

Parameters:

include – fields to include in new model
exclude – fields to exclude from new model, as with values this takes precedence over include
update – values to change/add in the new model. Note: the data is not validated before creating the new model: you should trust this data
deep – set to True to make a deep copy of the model

Returns:

new model instance

covers(value: ColumnValueType) → bool#

Return True if input value is already covered by HWM

Examples

column = Column(name="id")
table = Table(name="mydb.mytable", instance="postgres://db.host:5432")

old_hwm = ColumnHWM(column=column, source=table, value=1)

assert old_hwm.covers(0)  # 0 <= 1
assert old_hwm.covers(1)  # 1 <= 1
assert old_hwm.covers(0.5)  # 0.5 <= 1
assert not old_hwm.covers(2)  # 2 > 1

empty_hwm = ColumnHWM(column=column, source=table)

assert not empty_hwm.covers(0)  # non comparable with None
assert not empty_hwm.covers(1)  # non comparable with None
assert not empty_hwm.covers(0.5)  # non comparable with None
assert not empty_hwm.covers(2)  # non comparable with None

classmethod deserialize(inp: dict)#

Return HWM from dict representation

Returns:

resultHWM: Deserialized HWM

Examples

from etl_entities.old_hwm import IntHWM

assert IntHWM.deserialize(
    {
        "value": "1",
        "type": "int",
        "column": {"name": ..., "partition": ...},
        "source": ...,
        "process": ...,
    }
) == IntHWM(value=1, ...)

IntHWM.deserialize({"type": "date"})  # raises ValueError

dict(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) → DictStrAny#: Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.

json(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Callable[[Any], Any] | None = None, models_as_dict: bool = True, **dumps_kwargs: Any) → str#

Generate a JSON representation of the model, include and exclude arguments as per dict().

encoder is an optional function to supply as default to json.dumps(), other arguments as per json.dumps().

property name: str#

HWM column name

Returns:

valuestr: Column name

Examples

column = Column(name="id")
table = Table(name="mydb.mytable", instance="postgres://db.host:5432")

old_hwm = ColumnHWM(column=column, source=table, value=val)

assert old_hwm.name == "id"

property qualified_name: str#

Unique name of HWM

Returns:

valuestr: Qualified name

Examples

column = Column(name="id")
table = Table(name="mydb.mytable", instance="postgres://db.host:5432")

old_hwm = ColumnHWM(column=column, source=table, value=1)

assert (
    old_hwm.qualified_name
    == "id#mydb.mytable@postgres://db.host:5432#currentprocess@currenthost"
)

serialize() → dict#

Return dict representation of HWM

Returns:

resultdict: Serialized HWM

Examples

from etl_entities.old_hwm import IntHWM

old_hwm = IntHWM(value=1, ...)
assert old_hwm.serialize() == {
    "value": "1",
    "type": "int",
    "column": {"name": ..., "partition": ...},
    "source": ...,
    "process": ...,
}

set_value(value: ValueType) → HWM#

Replaces current HWM value with the passed one, and return HWM.

Note

Changes HWM value in place instead of returning new one

Returns:

resultHWM: Self

Examples

from etl_entities.old_hwm import IntHWM

old_hwm = IntHWM(value=1, ...)

old_hwm.set_value(2)
assert old_hwm.value == 2

update(value: ColumnValueType)#

Updates current HWM value with some implementation-specific logic, and return HWM.

Note

Changes HWM value in place

Returns:

resultColumnHWM: HWM copy with new value

Examples

from etl_entities.old_hwm import IntHWM

old_hwm = IntHWM(value=1, ...)

old_hwm.update(2)
assert old_hwm.value == 2

old_hwm.update(1)
assert old_hwm.value == 2  # value cannot decrease