Datetime HWM#
- class etl_entities.old_hwm.datetime_hwm.DateTimeHWM(*, source: <pydantic.v1.fields.DeferredType object at 0x7ffb12196c60>, value: ~datetime.datetime | None = None, modified_time: <pydantic.v1.fields.DeferredType object at 0x7ffb120b8290> = None, process: <pydantic.v1.fields.DeferredType object at 0x7ffb120b8bc0> = None, column: <pydantic.v1.fields.DeferredType object at 0x7ffb120bb1a0>)#
DateTime HWM type
Deprecated since version 2.0.0: Use
ColumnDateTimeHWM
instead- Parameters:
- column
etl_entities.source.db.column.Column
Column instance
- source
etl_entities.source.db.table.Table
Table instance
- value
datetime.datetime
orNone
, default:None
HWM value
- modified_time
datetime.datetime
, default: current datetime HWM value modification time
- process
etl_entities.process.process.Process
, default: current process Process instance
- column
Examples
from datetime import datetime from etl_entities.old_hwm import DateTimeHWM from etl_entities.source import Column, Table column = Column(name="id") table = Table(name="mydb.mytable", instance="postgres://db.host:5432") old_hwm = DateTimeHWM( column=column, source=table, value=datetime(year=2021, month=12, day=31, hour=11, minute=22, second=33), )
- __add__(value)#
Increase HWM value and return copy of HWM
- Parameters:
- value
Any
orNone
Should be compatible with
value
attribute type.For example, you cannot add
str
toint
value, but you can addint
toint
.
- value
- Returns:
- resultColumnHWM
HWM copy with new value
Examples
# assume val2 == val1 + inc hwm1 = ColumnHWM(value=val1, ...) hwm2 = ColumnHWM(value=val2, ...) # same as ColumnHWM(value=hwm1.value + inc, ...) assert hwm1 + inc == hwm2
- __bool__()#
Check if HWM value is set
- Returns:
- resultbool
False
ifvalue
isNone
,True
otherwise
Examples
from etl_entities.old_hwm import ColumnHWM old_hwm = ColumnHWM(value=1, ...) assert old_hwm # same as old_hwm.value is not None old_hwm = ColumnHWM(value=None, ...) assert not old_hwm
- __eq__(other)#
Checks equality of two HWM instances
- Parameters:
- other
etl_entities.old_hwm.datetime_hwm.DateTimeHWM
ordatetime.datetime
Should be comparable with
value
attribute type.You can compare two
etl_entities.old_hwm.datetime_hwm.DateTimeHWM
ordatetime
values.But you cannot compare
datetime
withint
value, as well as different HWM types, likeetl_entities.old_hwm.datetime_hwm.DateTimeHWM
andetl_entities.old_hwm.int_hwm.IntHWM
.
- other
- Returns:
- resultbool
True
if both inputs are the same,False
otherwise.
Examples
from datetime import datetime from etl_entities.old_hwm import DateTimeHWM hwm1 = DateTimeHWM( value=datetime(year=2021, month=12, day=30, hour=11, minute=22, second=33), ... ) hwm2 = DateTimeHWM( value=datetime(year=2021, month=12, day=31, hour=1, minute=11, second=22), ... ) assert hwm1 == hwm1 assert hwm1 != hwm2
- __lt__(other)#
Checks current HWM value is less than another one
- Parameters:
- other
etl_entities.old_hwm.datetime_hwm.DateTimeHWM
ordatetime.datetime
Should be comparable with
value
attribute type.You can compare two
etl_entities.old_hwm.datetime_hwm.DateTimeHWM
ordatetime
values.But you cannot compare
datetime
withint
value, as well as different HWM types, likeetl_entities.old_hwm.datetime_hwm.DateTimeHWM
andetl_entities.old_hwm.int_hwm.IntHWM
.Warning
You cannot compare HWMs if one of them has
None
value
- other
- Returns:
- resultbool
True
if current HWM value is less than provided value,False
otherwise.
Examples
from datetime import datetime from etl_entities.old_hwm import DateTimeHWM hwm1 = DateTimeHWM( value=datetime(year=2021, month=12, day=30, hour=11, minute=22, second=33), ... ) hwm2 = DateTimeHWM( value=datetime(year=2021, month=12, day=31, hour=00, minute=11, second=22), ... ) assert hwm1 < hwm2 assert hwm2 > hwm1 assert hwm1 < datetime(year=2021, month=12, day=31, hour=1, minute=11, second=22) assert hwm1 > datetime(year=2021, month=12, day=30, hour=11, minute=22, second=33) hwm3 = DateTimeHWM(value=None, ...) assert hwm1 < hwm3 # will raise TypeError assert hwm1 < None # same thing
- __sub__(value)#
Decrease HWM value, and return copy of HWM
- Parameters:
- value
Any
orNone
Should be compatible with
value
attribute type.For example, you cannot subtract
str
fromint
value, but you can subtractint
fromint
.
- value
- Returns:
- resultColumnHWM
HWM copy with new value
Examples
# assume val2 == val1 - dec hwm1 = ColumnHWM(value=val1, ...) hwm2 = ColumnHWM(value=val2, ...) # same as ColumnHWM(value=hwm1.value - dec, ...) assert hwm1 - dec == hwm2
- copy(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, update: DictStrAny | None = None, deep: bool = False) Model #
Duplicate a model, optionally choose which fields to include, exclude and change.
- Parameters:
include – fields to include in new model
exclude – fields to exclude from new model, as with values this takes precedence over include
update – values to change/add in the new model. Note: the data is not validated before creating the new model: you should trust this data
deep – set to True to make a deep copy of the model
- Returns:
new model instance
- covers(value: ColumnValueType) bool #
Return
True
if input value is already covered by HWMExamples
column = Column(name="id") table = Table(name="mydb.mytable", instance="postgres://db.host:5432") old_hwm = ColumnHWM(column=column, source=table, value=1) assert old_hwm.covers(0) # 0 <= 1 assert old_hwm.covers(1) # 1 <= 1 assert old_hwm.covers(0.5) # 0.5 <= 1 assert not old_hwm.covers(2) # 2 > 1 empty_hwm = ColumnHWM(column=column, source=table) assert not empty_hwm.covers(0) # non comparable with None assert not empty_hwm.covers(1) # non comparable with None assert not empty_hwm.covers(0.5) # non comparable with None assert not empty_hwm.covers(2) # non comparable with None
- classmethod deserialize(inp: dict)#
Return HWM from dict representation
- Returns:
- resultHWM
Deserialized HWM
Examples
from etl_entities.old_hwm import IntHWM assert IntHWM.deserialize( { "value": "1", "type": "int", "column": {"name": ..., "partition": ...}, "source": ..., "process": ..., } ) == IntHWM(value=1, ...) IntHWM.deserialize({"type": "date"}) # raises ValueError
- dict(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) DictStrAny #
Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.
- json(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Callable[[Any], Any] | None = None, models_as_dict: bool = True, **dumps_kwargs: Any) str #
Generate a JSON representation of the model, include and exclude arguments as per dict().
encoder is an optional function to supply as default to json.dumps(), other arguments as per json.dumps().
- property name: str#
HWM column name
- Returns:
- valuestr
Column name
Examples
column = Column(name="id") table = Table(name="mydb.mytable", instance="postgres://db.host:5432") old_hwm = ColumnHWM(column=column, source=table, value=val) assert old_hwm.name == "id"
- property qualified_name: str#
Unique name of HWM
- Returns:
- valuestr
Qualified name
Examples
column = Column(name="id") table = Table(name="mydb.mytable", instance="postgres://db.host:5432") old_hwm = ColumnHWM(column=column, source=table, value=1) assert ( old_hwm.qualified_name == "id#mydb.mytable@postgres://db.host:5432#currentprocess@currenthost" )
- serialize() dict #
Return dict representation of HWM
- Returns:
- resultdict
Serialized HWM
Examples
from etl_entities.old_hwm import IntHWM old_hwm = IntHWM(value=1, ...) assert old_hwm.serialize() == { "value": "1", "type": "int", "column": {"name": ..., "partition": ...}, "source": ..., "process": ..., }
- set_value(value: ValueType) HWM #
Replaces current HWM value with the passed one, and return HWM.
Note
Changes HWM value in place instead of returning new one
- Returns:
- resultHWM
Self
Examples
from etl_entities.old_hwm import IntHWM old_hwm = IntHWM(value=1, ...) old_hwm.set_value(2) assert old_hwm.value == 2
- update(value: ColumnValueType)#
Updates current HWM value with some implementation-specific logic, and return HWM.
Note
Changes HWM value in place
- Returns:
- resultColumnHWM
HWM copy with new value
Examples
from etl_entities.old_hwm import IntHWM old_hwm = IntHWM(value=1, ...) old_hwm.update(2) assert old_hwm.value == 2 old_hwm.update(1) assert old_hwm.value == 2 # value cannot decrease