Data Tranformation - Data Preprocessing
Data Tranformation - Data Preprocessing
"{
""specversion"": ""0.1"",
""type"": ""activity"",
""source"": null,
""subject"": ""ACTIVITY"",
""id"": ""e0731"",
""correlation"": null,
""causation"": null,
""comexampleextension1"": null,
""time"": ""2023-03-01 00:00:00+05:30"",
""action"": ""CREATE"",
""datacontenttype"": ""application/json"",
""membership_number"": ""mem128"",
""data"": {
""activity"": {
""memberActivity"": {
""membershipnumber"": ""mem128"",
""programCode"": ""SKW"",
""activityNumber"": ""act1248"",
""givenName"": ""aaa"",
""familyName"": ""fff"",
""description"": ""aaa bbb ccc"",
""activityDate"": ""01-03-2023 00:00:00"",
""memberActivityType"": ""AA"",
""status"": ""P"",
""cancellationActivityNumber"": ""null"",
""activityCode"": ""actcod1248"",
""partnerCode"": ""EK"",
""userCode"": ""user128"",
""processDate"": ""01-03-2023 00:00:00""
},
""memberPointTransaction"": {
""programCode"": ""SKW"",
""activityNumber"": ""act1248"",
""transactionNumber"": ""tran1248"",
""pointType"": ""skywardmiles"",
""transactionType"": ""AA"",
""transactionDate"": ""01-03-2023 00:00:00"",
""points"": ""25000"",
""transactionActivityType"": ""AA"",
""bonusCode"": """",
""creationDate"": ""01-03-2023 00:00:00""
},
""tierChangeActivity"": {
""programCode"": """",
""activityNumber"": """",
""currentTier"": """",
""previousTier"": """",
""tierExpiryDate"": """",
""tierChangeType"": """",
""reasonCode"": """",
""remarks"": """",
""tierActivationDate"": """",
""action"": """"
},
""memberAccrualActivity"": {
""accrualActivityReferenceNumber"": ""aaact1248"",
""billnumber"": ""bil1238"",
""accrualactivitypartnerattribute"": {
""attributecode"": """",
""attributevalue"": """"
},
""accrualActivity"": {
""activityReferenceNumber"": ""aaact1248"",
""membershipNumber"": ""mem128"",
""activitySource"": """",
""partnerCode"": ""EK"",
""givenName"": ""aaa"",
""familyName"": ""fff"",
""businessType"": """",
""activityDate"": ""01-03-2023 00:00:00"",
""status"": """",
""amountSpent"": ""2000"",
""reasonCode"": """",
""remarks"": """",
""pointValue"": """",
""userCode"": ""user128"",
""activityDescription"": """",
""billingStatus"": """",
""revisionStatus"": """",
""internalIdentifier"": """",
""artefactType"": """",
""artefactNum"": """",
""action"": """"
},
""airAccrualActivity"": {
""activityReferenceNumber"": ""aaact1248"",
""airlineIdentifier"": """",
""flightNumber"": ""EK001"",
""boardedBookingClass"": ""J"",
""originalBookingClass"": ""J"",
""cabinClass"": ""Business"",
""origin"": ""TRV"",
""destination"": ""DXB"",
""ssrCode"": """",
""ticketNumber"": ""t1237"",
""seatNumber"": """",
""milesFlown"": """",
""ticketFirstName"": """",
""ticketLastName"": """",
""ticketTitle"": """",
""ticketNameVldnOverride"": """",
""marketingCarrierCode"": ""EK"",
""marketingFlightNumber"": ""EK001"",
""operatingCarrierCode"": ""EK"",
""operatingFlightNumber"": ""EK001"",
""isTicketedOnHost"": """",
""pnr"": ""pnr123"",
""ticketIssuedDate"": ""02-02-2022 00:00:00""
},
""nonAirAccrualActivity"": {
""activityReferenceNumber"": """",
""billNumber"": """",
""action"": """"
}
}
}
}
}
"{
""specversion"": ""0.1"",
""type"": ""account"",
""source"": null,
""subject"": ""ACCOUNT"",
""id"": ""e0727"",
""correlation"": null,
""causation"": null,
""comexampleextension1"": null,
""time"": ""2023-03-01 00:00:00+05:30"",
""action"": ""UPDATE"",
""datacontenttype"": ""application/json"",
""membership_number"": ""mem128"",
""data"": {
""account"": {
""member"": {
""programCode"": ""PRG"",
""membershipnumber"": ""mem128"",
""membershipStatus"": ""A"",
""userCode"": ""user128"",
""customerNumber"": ""cus128"",
""webLogin"": ""T""
},
""account"": {
""accountStatus"": ""A"",
""tierCode"": ""Silver"",
""tierExpiryDate"": ""30-02-2024 00:00:00"",
""enrollmentSource"": ""P"",
""enrollmentSourceCode"": ""EK"",
""enrollmentDate"": ""27-02-2023 00:00:00"",
""suspendFlag"": ""F"",
""accountCreationDate"": ""30-02-2023 00:00:00""
},
""accountdetail"": {
""pointType"": ""skywardmiles"",
""points"": ""54000"",
""totalAccruedPoints"": ""54000"",
""totalRedeemedPoints"": 0,
""totalExpiredPoints"": 0,
""programCode"": ""PRG""
}
}
}
}
"
"{
""name"": ""offer_creation"",
""id"": ""e00710"",
""time"": ""2023-01-30 00:00:00+05:30"",
""action"": ""CREATE"",
""offer_id"": ""O100"",
""data"": {
""offerTypeId"": 2,
""offerName"": ""sdf"",
""offerDescription"": ""sdf"",
""offerReferenceCode"": ""f"",
""offerCategory"": ""commercial"",
""offerStartDate"": ""2023-02-01T18:30:00.000Z"",
""offerEndDate"": ""2023-05-20T18:30:00.000Z"",
""createdBy"": ""ui"",
""modifiedBy"": ""ui"",
""blackoutDate"": [""2023-02-07T18:30:00.000Z""],
""workflowDetails"": { ""status"": ""ACTIVE"" },
""rules"": {
""name"": ""sdf"",
""description"": ""sdf"",
""rules"": [
{
""ruleId"": ""R100"",
""rule"": {
""when"": {
""all"": [
{
""lhs"": ""$member.tier$"",
""operation"": ""IN"",
""rhs"": ""Silver""
},
{
""lhs"": ""$member.country_residence$"",
""operation"": ""IN"",
""rhs"": ""India""
},
{ ""lhs"": ""$member.gender$"", ""operation"": ""IN"", ""rhs"": ""F"" },
{
""lhs"": ""$member.historic_criteria.member_activity_period7.count$"",
""operation"": ""GT"",
""rhs"": ""1""
},
{
""lhs"": ""$member.historic_criteria.member_activity_period7.interval$"",
""operation"": ""LTE"",
""rhs"": ""3year""
},
{
""lhs"": ""$member.historic_criteria.member_activity_period7.activity_type$"",
""operation"": ""EQ"",
""rhs"": ""AA""
}
]
},
""then"": {
""data"": {
""offer_id"": ""O100"",
""rule_id"": ""R100"",
""offer_benefit"": ""B100"",
""membership_number"": ""NULL""
}
}
}
},
{
""ruleId"": ""R101"",
""rule"": {
""when"": {
""all"": [
{
""lhs"": ""$member.tier$"",
""operation"": ""IN"",
""rhs"": ""Silver""
},
{
""lhs"": ""$member.country_residence$"",
""operation"": ""IN"",
""rhs"": ""UAE""
},
{ ""lhs"": ""$member.gender$"", ""operation"": ""IN"", ""rhs"": ""M"" },
{
""lhs"": ""$member.historic_criteria.member_activity_period8.count$"",
""operation"": ""GTE"",
""rhs"": ""1""
},
{
""lhs"": ""$member.historic_criteria.member_activity_period8.interval$"",
""operation"": ""LTE"",
""rhs"": ""2year""
},
{
""lhs"": ""$member.historic_criteria.member_activity_period8.activity_type$"",
""operation"": ""EQ"",
""rhs"": ""AA""
}
]
},
""then"": {
""data"": {
""offer_id"": ""O100"",
""rule_id"": ""R101"",
""offer_benefit"": ""B101"",
""membership_number"": ""NULL""
}
}
}
}
]
},
""inputData"": {
""member"": {
""membership_number"": ""NULL"",
""tier"": ""NULL"",
""country_residence"": ""NULL"",
""gender"": ""NULL"",
""historic_criteria"": {
""member_activity_period7"": {
""count"": ""activity_type"",
""interval"": ""3year"",
""activity_type"": ""AA""
},
""member_activity_period8"": {
""count"": ""activity_type"",
""interval"": ""2year"",
""activity_type"": ""AA""
}
}
}
}
}
}
"
Input dataframe2:
Df1 Df2
specversion name
type id
source time
subject action
Offer_id O100
name
data {"accrual {"account {"offerTy
ActivityRe ": peId": 2,
ferenceN {"membe "offerNa
umber": r": me":
"aaact124 {"progra "sdf",
8", mCode": "offerDes
"billnumb "PRG", cription":
er": "member "sdf",
"bil1238", shipnumb "offerRef
"accruala er": erenceCo
ctivitypar "mem128 de": "f",
tnerattrib ", "offerCat
ute": "member egory":
membres mem128
{"attribut mem128
shipStatu "commer
ecode": s": "A", cial",
offer_id
dataconte applicatio
"", applicatio
"userCod "offerStar
data
ntetype n/json
"attribute n/json
e": tDate":
action CREATE
value": UPDATE
"user128" CREATE
"2023-02-
""}, , 01T18:30:
time 2023-03-
"accrualA 2023-03-
"custome 2023-01-
00.000Z",
01
ctivity": 01
rNumber" 30
"offerEnd
00:00:00+
{"activity :00:00:00+ 00:00:00+
Date":
05:30
Referenc 05:30
"cus128", 05:30
"2023-05-
Comeexa
mpleexte eNumber "webLogi 20T18:30:
causation
nsion1 ": n": "T"}, 00.000Z",
"aaact124 "account" "createdB
comexampleextension1
membership_number
correlatio 8", : y": "ui",
n "member {"account "modified
datacontenttype
id shipNum
E0731 Status":
e0727 By":
e00710"ui",
ber": "A", "blackout
correlation
causation subject "mem128
ACTIVITY "tierCode
ACCOUNT Date":
action
", ": ["2023-
time
data
id source "activityS "Silver", 02-
ource": "tierExpir 07T18:30:
type activity
"", account
yDate": 00.000Z"]
"partnerC "30-02- ,
specvisio 0.1
ode": 0.1
2024 "workflo
n "EK", 00:00:00" wDetails"
"givenNa , :
Source code: