== Physical Plan ==
AdaptiveSparkPlan (11)
+- == Final Plan ==
ResultQueryStage (7), Statistics(sizeInBytes=8.0 EiB)
+- ObjectHashAggregate (6)
+- ShuffleQueryStage (5), Statistics(sizeInBytes=10.7 KiB, rowCount=50)
+- Exchange (4)
+- ObjectHashAggregate (3)
+- * Project (2)
+- * Scan ExistingRDD Delta Table State #2 - hdlfs://f1cc4c74-cdd4-440e-abc7-4e0157628b42.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log (1)
+- == Initial Plan ==
ObjectHashAggregate (10)
+- Exchange (9)
+- ObjectHashAggregate (8)
+- Project (2)
+- Scan ExistingRDD Delta Table State #2 - hdlfs://f1cc4c74-cdd4-440e-abc7-4e0157628b42.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log (1)
(1) Scan ExistingRDD Delta Table State #2 - hdlfs://f1cc4c74-cdd4-440e-abc7-4e0157628b42.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log [codegen id : 1]
Output [10]: [txn#21518599, add#21518600, remove#21518601, metaData#21518602, protocol#21518603, cdc#21518604, checkpointMetadata#21518605, sidecar#21518606, domainMetadata#21518607, commitInfo#21518608]
Arguments: [txn#21518599, add#21518600, remove#21518601, metaData#21518602, protocol#21518603, cdc#21518604, checkpointMetadata#21518605, sidecar#21518606, domainMetadata#21518607, commitInfo#21518608], Delta Table State #2 - hdlfs://f1cc4c74-cdd4-440e-abc7-4e0157628b42.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log MapPartitionsRDD[1241349] at $anonfun$recordDeltaOperationInternal$1 at DatabricksLogging.scala:128, ExistingRDD, UnknownPartitioning(0)
(2) Project [codegen id : 1]
Output [6]: [txn#21518599, add#21518600, remove#21518601, metaData#21518602, protocol#21518603, domainMetadata#21518607]
Input [10]: [txn#21518599, add#21518600, remove#21518601, metaData#21518602, protocol#21518603, cdc#21518604, checkpointMetadata#21518605, sidecar#21518606, domainMetadata#21518607, commitInfo#21518608]
(3) ObjectHashAggregate
Input [6]: [txn#21518599, add#21518600, remove#21518601, metaData#21518602, protocol#21518603, domainMetadata#21518607]
Keys: []
Functions [10]: [partial_collect_set(domainMetadata#21518607, 0, 0), partial_collect_set(txn#21518599, 0, 0), partial_count(protocol#21518603), partial_sum(add#21518600.size), partial_last(metaData#21518602, true), partial_count(metaData#21518602), partial_last(protocol#21518603, true), partial_count(remove#21518601), partial_count(add#21518600), partial_count(txn#21518599)]
Aggregate Attributes [12]: [buf#21518709, buf#21518710, count#21518711L, sum#21518712L, last#21518713, valueSet#21518714, count#21518715L, last#21518716, valueSet#21518717, count#21518718L, count#21518719L, count#21518720L]
Results [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
(4) Exchange
Input [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21256006]
(5) ShuffleQueryStage
Output [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
Arguments: 0
(6) ObjectHashAggregate
Input [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
Keys: []
Functions [10]: [collect_set(domainMetadata#21518607, 0, 0), collect_set(txn#21518599, 0, 0), count(protocol#21518603), sum(add#21518600.size), last(metaData#21518602, true), count(metaData#21518602), last(protocol#21518603, true), count(remove#21518601), count(add#21518600), count(txn#21518599)]
Aggregate Attributes [10]: [collect_set(domainMetadata#21518607, 0, 0)#21518655, collect_set(txn#21518599, 0, 0)#21518656, count(protocol#21518603)#21518657L, sum(add#21518600.size)#21518658L, last(metaData#21518602) ignore nulls#21518659, count(metaData#21518602)#21518660L, last(protocol#21518603) ignore nulls#21518661, count(remove#21518601)#21518662L, count(add#21518600)#21518663L, count(txn#21518599)#21518664L]
Results [14]: [collect_set(domainMetadata#21518607, 0, 0)#21518655 AS domainMetadata#21518640, null AS fileSizeHistogram#21518641, collect_set(txn#21518599, 0, 0)#21518656 AS setTransactions#21518642, null AS numDeletedRecordsOpt#21518643, count(protocol#21518603)#21518657L AS numOfProtocol#21518644L, coalesce(sum(add#21518600.size)#21518658L, 0) AS sizeInBytes#21518645L, last(metaData#21518602) ignore nulls#21518659 AS metadata#21518646, count(metaData#21518602)#21518660L AS numOfMetadata#21518647L, null AS deletedRecordCountsHistogramOpt#21518648, null AS numDeletionVectorsOpt#21518649, last(protocol#21518603) ignore nulls#21518661 AS protocol#21518650, count(remove#21518601)#21518662L AS numOfRemoves#21518651L, count(add#21518600)#21518663L AS numOfFiles#21518652L, count(txn#21518599)#21518664L AS numOfSetTransactions#21518653L]
(7) ResultQueryStage
Output [14]: [domainMetadata#21518640, fileSizeHistogram#21518641, setTransactions#21518642, numDeletedRecordsOpt#21518643, numOfProtocol#21518644L, sizeInBytes#21518645L, metadata#21518646, numOfMetadata#21518647L, deletedRecordCountsHistogramOpt#21518648, numDeletionVectorsOpt#21518649, protocol#21518650, numOfRemoves#21518651L, numOfFiles#21518652L, numOfSetTransactions#21518653L]
Arguments: 1
(8) ObjectHashAggregate
Input [6]: [txn#21518599, add#21518600, remove#21518601, metaData#21518602, protocol#21518603, domainMetadata#21518607]
Keys: []
Functions [10]: [partial_collect_set(domainMetadata#21518607, 0, 0), partial_collect_set(txn#21518599, 0, 0), partial_count(protocol#21518603), partial_sum(add#21518600.size), partial_last(metaData#21518602, true), partial_count(metaData#21518602), partial_last(protocol#21518603, true), partial_count(remove#21518601), partial_count(add#21518600), partial_count(txn#21518599)]
Aggregate Attributes [12]: [buf#21518709, buf#21518710, count#21518711L, sum#21518712L, last#21518713, valueSet#21518714, count#21518715L, last#21518716, valueSet#21518717, count#21518718L, count#21518719L, count#21518720L]
Results [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
(9) Exchange
Input [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21255995]
(10) ObjectHashAggregate
Input [12]: [buf#21518721, buf#21518722, count#21518723L, sum#21518724L, last#21518725, valueSet#21518726, count#21518727L, last#21518728, valueSet#21518729, count#21518730L, count#21518731L, count#21518732L]
Keys: []
Functions [10]: [collect_set(domainMetadata#21518607, 0, 0), collect_set(txn#21518599, 0, 0), count(protocol#21518603), sum(add#21518600.size), last(metaData#21518602, true), count(metaData#21518602), last(protocol#21518603, true), count(remove#21518601), count(add#21518600), count(txn#21518599)]
Aggregate Attributes [10]: [collect_set(domainMetadata#21518607, 0, 0)#21518655, collect_set(txn#21518599, 0, 0)#21518656, count(protocol#21518603)#21518657L, sum(add#21518600.size)#21518658L, last(metaData#21518602) ignore nulls#21518659, count(metaData#21518602)#21518660L, last(protocol#21518603) ignore nulls#21518661, count(remove#21518601)#21518662L, count(add#21518600)#21518663L, count(txn#21518599)#21518664L]
Results [14]: [collect_set(domainMetadata#21518607, 0, 0)#21518655 AS domainMetadata#21518640, null AS fileSizeHistogram#21518641, collect_set(txn#21518599, 0, 0)#21518656 AS setTransactions#21518642, null AS numDeletedRecordsOpt#21518643, count(protocol#21518603)#21518657L AS numOfProtocol#21518644L, coalesce(sum(add#21518600.size)#21518658L, 0) AS sizeInBytes#21518645L, last(metaData#21518602) ignore nulls#21518659 AS metadata#21518646, count(metaData#21518602)#21518660L AS numOfMetadata#21518647L, null AS deletedRecordCountsHistogramOpt#21518648, null AS numDeletionVectorsOpt#21518649, last(protocol#21518603) ignore nulls#21518661 AS protocol#21518650, count(remove#21518601)#21518662L AS numOfRemoves#21518651L, count(add#21518600)#21518663L AS numOfFiles#21518652L, count(txn#21518599)#21518664L AS numOfSetTransactions#21518653L]
(11) AdaptiveSparkPlan
Output [14]: [domainMetadata#21518640, fileSizeHistogram#21518641, setTransactions#21518642, numDeletedRecordsOpt#21518643, numOfProtocol#21518644L, sizeInBytes#21518645L, metadata#21518646, numOfMetadata#21518647L, deletedRecordCountsHistogramOpt#21518648, numDeletionVectorsOpt#21518649, protocol#21518650, numOfRemoves#21518651L, numOfFiles#21518652L, numOfSetTransactions#21518653L]
Arguments: isFinalPlan=true