Alessandro Duico
02/02/2022, 3:15 PMclass ServicesnowApi(HttpStream):
url_base = "<https://xxx.service-now.com/api/now/v1/>"
# Set this as a noop.
primary_key = None
# Save the state every 100 records
state_checkpoint_interval = 100
page_size = 100
cursor_field = "sys_updated_on"
def __init__(self, limit: str, sys_created_from: str, **kwargs):
super().__init__(**kwargs)
# Here's where we set the variable from our input to pass it down to the source.
self.limit = limit
self.sys_created_from = sys_created_from
def path(self, **kwargs) -> str:
# This defines the path to the endpoint that we want to hit.
limit = self.limit
sys_created_from = self.sys_created_from
return f"table/incident?sysparm_offset=0&sysparm_limit={limit}&sysparm_query=sys_created_on>={sys_created_from} 00:00^active=ISNOTEMPTY"
def request_params(
self,
stream_state: Mapping[str, Any],
stream_slice: Mapping[str, Any] = None,
next_page_token: Mapping[str, Any] = None,
) -> MutableMapping[str, Any]:
limit = self.limit
sys_created_from = self.sys_created_from
return {"limit": limit, "sys_created_from":sys_created_from}
def parse_response(
self,
response: requests.Response,
stream_state: Mapping[str, Any],
stream_slice: Mapping[str, Any] = None,
next_page_token: Mapping[str, Any] = None,
) -> Iterable[Mapping]:
result = response.json()['result']
return result
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
return None
Daniel Eduardo Portugal Revilla
02/02/2022, 4:00 PMclass Events(IncrementalAmplitudeStream):
cursor_field = "event_time"
date_template = "%Y%m%dT%H"
primary_key = "uuid"
state_checkpoint_interval = 1000
time_interval = {"days": 3}
Nathan Gille
02/03/2022, 4:43 AMif (recordsRead % 1000 == 0) {
<http://LOGGER.info|LOGGER.info>("Records read: {}", recordsRead);
}
And I think the get_updated_state is called here for every 100 recordsFamezilla Channel
02/03/2022, 6:01 AMstate_checkpoint_interval = 100
page_size = 100
but nothingstate_checkpoint_interval = 100
the last job has 10000 records and on S3 there is only one .json with 64MB.
I imagine the expected output is 10000/100 files.Khristina Rustanovich
02/03/2022, 3:36 PMMaxime edfeed
02/04/2022, 4:45 AMstate_checkpoint_interval = 100
on S3 there is only one file with 10000 records.cursor_field = "sys_updated_on"
but i can not see it here