mc-data-pipeline - Environment

Runtime Information

Name	Value
Java Home	/opt/java/openjdk
Java Version	17.0.12 (Eclipse Adoptium)
Scala Version	version 2.12.18

Spark Properties

Name	Value
spark.app.id	local-1765186458233
spark.app.initial.jar.urls	spark://07f6cd05c5e7:42015/jars/data_pipeline.jar
spark.app.name	mc-data-pipeline
spark.app.startTime	1765186451103
spark.app.submitTime	1765186446144
spark.driver.extraJavaOptions	-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false
spark.driver.host	07f6cd05c5e7
spark.driver.port	42015
spark.executor.extraJavaOptions	-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false
spark.executor.id	driver
spark.jars	file:/opt/spark/data_pipeline.jar
spark.master	local[*]
spark.scheduler.mode	FIFO
spark.sql.shuffle.partitions	4
spark.streaming.stopGracefullyOnShutdown	true
spark.submit.deployMode	client
spark.submit.pyFiles

Resource Profiles

Resource Profile Id	Resource Profile Contents
0	Executor Reqs: cores: [amount: 1] memory: [amount: 1024] offHeap: [amount: 0] Task Reqs: cpus: [amount: 1.0]

Hadoop Properties

Name	Value
adl.feature.ownerandgroup.enableupn	false
adl.http.timeout	-1
dfs.client.ignore.namenode.default.kms.uri	false
dfs.ha.fencing.ssh.connect-timeout	30000
file.blocksize	67108864
file.bytes-per-checksum	512
file.client-write-packet-size	65536
file.replication	1
file.stream-buffer-size	4096
fs.AbstractFileSystem.abfs.impl	org.apache.hadoop.fs.azurebfs.Abfs
fs.AbstractFileSystem.abfss.impl	org.apache.hadoop.fs.azurebfs.Abfss
fs.AbstractFileSystem.adl.impl	org.apache.hadoop.fs.adl.Adl
fs.AbstractFileSystem.file.impl	org.apache.hadoop.fs.local.LocalFs
fs.AbstractFileSystem.ftp.impl	org.apache.hadoop.fs.ftp.FtpFs
fs.AbstractFileSystem.gs.impl	com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS
fs.AbstractFileSystem.har.impl	org.apache.hadoop.fs.HarFs
fs.AbstractFileSystem.hdfs.impl	org.apache.hadoop.fs.Hdfs
fs.AbstractFileSystem.s3a.impl	org.apache.hadoop.fs.s3a.S3A
fs.AbstractFileSystem.swebhdfs.impl	org.apache.hadoop.fs.SWebHdfs
fs.AbstractFileSystem.viewfs.impl	org.apache.hadoop.fs.viewfs.ViewFs
fs.AbstractFileSystem.wasb.impl	org.apache.hadoop.fs.azure.Wasb
fs.AbstractFileSystem.wasbs.impl	org.apache.hadoop.fs.azure.Wasbs
fs.AbstractFileSystem.webhdfs.impl	org.apache.hadoop.fs.WebHdfs
fs.abfs.impl	org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem
fs.abfss.impl	org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem
fs.adl.impl	org.apache.hadoop.fs.adl.AdlFileSystem
fs.adl.oauth2.access.token.provider.type	*********(redacted)
fs.automatic.close	true
fs.azure.authorization	false
fs.azure.authorization.caching.enable	true
fs.azure.buffer.dir	${hadoop.tmp.dir}/abfs
fs.azure.local.sas.key.mode	false
fs.azure.sas.expiry.period	90d
fs.azure.saskey.usecontainersaskeyforallaccess	true
fs.azure.secure.mode	false
fs.azure.user.agent.prefix	unknown
fs.client.resolve.remote.symlinks	true
fs.client.resolve.topology.enabled	false
fs.defaultFS	file:///
fs.df.interval	60000
fs.du.interval	600000
fs.ftp.data.connection.mode	ACTIVE_LOCAL_DATA_CONNECTION_MODE
fs.ftp.host	0.0.0.0
fs.ftp.host.port	21
fs.ftp.impl	org.apache.hadoop.fs.ftp.FTPFileSystem
fs.ftp.timeout	0
fs.ftp.transfer.mode	BLOCK_TRANSFER_MODE
fs.getspaceused.jitterMillis	60000
fs.har.impl.disable.cache	true
fs.permissions.umask-mode	022
fs.s3a.accesspoint.required	false
fs.s3a.assumed.role.credentials.provider	org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
fs.s3a.assumed.role.session.duration	30m
fs.s3a.attempts.maximum	20
fs.s3a.aws.credentials.provider	org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, com.amazonaws.auth.EnvironmentVariableCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
fs.s3a.block.size	32M
fs.s3a.buffer.dir	${hadoop.tmp.dir}/s3a
fs.s3a.change.detection.mode	server
fs.s3a.change.detection.source	etag
fs.s3a.change.detection.version.required	true
fs.s3a.committer.abort.pending.uploads	true
fs.s3a.committer.magic.enabled	true
fs.s3a.committer.name	file
fs.s3a.committer.staging.conflict-mode	append
fs.s3a.committer.staging.tmp.path	tmp/staging
fs.s3a.committer.staging.unique-filenames	true
fs.s3a.committer.threads	8
fs.s3a.connection.establish.timeout	5000
fs.s3a.connection.maximum	96
fs.s3a.connection.request.timeout	0
fs.s3a.connection.ssl.enabled	true
fs.s3a.connection.timeout	200000
fs.s3a.downgrade.syncable.exceptions	true
fs.s3a.endpoint	s3.amazonaws.com
fs.s3a.etag.checksum.enabled	false
fs.s3a.executor.capacity	16
fs.s3a.fast.upload.active.blocks	4
fs.s3a.fast.upload.buffer	disk
fs.s3a.impl	org.apache.hadoop.fs.s3a.S3AFileSystem
fs.s3a.list.version	2
fs.s3a.max.total.tasks	32
fs.s3a.metadatastore.authoritative	false
fs.s3a.metadatastore.fail.on.write.error	true
fs.s3a.metadatastore.impl	org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore
fs.s3a.metadatastore.metadata.ttl	15m
fs.s3a.multiobjectdelete.enable	true
fs.s3a.multipart.purge	false
fs.s3a.multipart.purge.age	86400
fs.s3a.multipart.size	64M
fs.s3a.multipart.threshold	128M
fs.s3a.paging.maximum	5000
fs.s3a.path.style.access	false
fs.s3a.readahead.range	64K
fs.s3a.retry.interval	500ms
fs.s3a.retry.limit	7
fs.s3a.retry.throttle.interval	100ms
fs.s3a.retry.throttle.limit	20
fs.s3a.s3guard.cli.prune.age	86400000
fs.s3a.s3guard.consistency.retry.interval	2s
fs.s3a.s3guard.consistency.retry.limit	7
fs.s3a.s3guard.ddb.background.sleep	25ms
fs.s3a.s3guard.ddb.max.retries	9
fs.s3a.s3guard.ddb.table.capacity.read	0
fs.s3a.s3guard.ddb.table.capacity.write	0
fs.s3a.s3guard.ddb.table.create	false
fs.s3a.s3guard.ddb.table.sse.enabled	false
fs.s3a.s3guard.ddb.throttle.retry.interval	100ms
fs.s3a.select.enabled	true
fs.s3a.select.errors.include.sql	false
fs.s3a.select.input.compression	none
fs.s3a.select.input.csv.comment.marker	#
fs.s3a.select.input.csv.field.delimiter	,
fs.s3a.select.input.csv.header	none
fs.s3a.select.input.csv.quote.character	"
fs.s3a.select.input.csv.quote.escape.character	\\
fs.s3a.select.input.csv.record.delimiter	\n
fs.s3a.select.output.csv.field.delimiter	,
fs.s3a.select.output.csv.quote.character	"
fs.s3a.select.output.csv.quote.escape.character	\\
fs.s3a.select.output.csv.quote.fields	always
fs.s3a.select.output.csv.record.delimiter	\n
fs.s3a.socket.recv.buffer	8192
fs.s3a.socket.send.buffer	8192
fs.s3a.ssl.channel.mode	default_jsse
fs.s3a.threads.keepalivetime	60
fs.s3a.threads.max	64
fs.swift.impl	org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
fs.trash.checkpoint.interval	0
fs.trash.interval	0
fs.viewfs.overload.scheme.target.abfs.impl	org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem
fs.viewfs.overload.scheme.target.abfss.impl	org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem
fs.viewfs.overload.scheme.target.file.impl	org.apache.hadoop.fs.LocalFileSystem
fs.viewfs.overload.scheme.target.ftp.impl	org.apache.hadoop.fs.ftp.FTPFileSystem
fs.viewfs.overload.scheme.target.gs.impl	com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS
fs.viewfs.overload.scheme.target.hdfs.impl	org.apache.hadoop.hdfs.DistributedFileSystem
fs.viewfs.overload.scheme.target.http.impl	org.apache.hadoop.fs.http.HttpFileSystem
fs.viewfs.overload.scheme.target.https.impl	org.apache.hadoop.fs.http.HttpsFileSystem
fs.viewfs.overload.scheme.target.o3fs.impl	org.apache.hadoop.fs.ozone.OzoneFileSystem
fs.viewfs.overload.scheme.target.ofs.impl	org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
fs.viewfs.overload.scheme.target.oss.impl	org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
fs.viewfs.overload.scheme.target.s3a.impl	org.apache.hadoop.fs.s3a.S3AFileSystem
fs.viewfs.overload.scheme.target.swebhdfs.impl	org.apache.hadoop.hdfs.web.SWebHdfsFileSystem
fs.viewfs.overload.scheme.target.swift.impl	org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
fs.viewfs.overload.scheme.target.wasb.impl	org.apache.hadoop.fs.azure.NativeAzureFileSystem
fs.viewfs.overload.scheme.target.webhdfs.impl	org.apache.hadoop.hdfs.web.WebHdfsFileSystem
fs.viewfs.rename.strategy	SAME_MOUNTPOINT
fs.wasb.impl	org.apache.hadoop.fs.azure.NativeAzureFileSystem
fs.wasbs.impl	org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure
ftp.blocksize	67108864
ftp.bytes-per-checksum	512
ftp.client-write-packet-size	65536
ftp.replication	3
ftp.stream-buffer-size	4096
ha.failover-controller.active-standby-elector.zk.op.retries	3
ha.failover-controller.cli-check.rpc-timeout.ms	20000
ha.failover-controller.graceful-fence.connection.retries	1
ha.failover-controller.graceful-fence.rpc-timeout.ms	5000
ha.failover-controller.new-active.rpc-timeout.ms	60000
ha.health-monitor.check-interval.ms	1000
ha.health-monitor.connect-retry-interval.ms	1000
ha.health-monitor.rpc-timeout.ms	45000
ha.health-monitor.rpc.connect.max.retries	1
ha.health-monitor.sleep-after-disconnect.ms	1000
ha.zookeeper.acl	world:anyone:rwcda
ha.zookeeper.parent-znode	/hadoop-ha
ha.zookeeper.session-timeout.ms	10000
hadoop.caller.context.enabled	false
hadoop.caller.context.max.size	128
hadoop.caller.context.signature.max.size	40
hadoop.common.configuration.version	3.0.0
hadoop.domainname.resolver.impl	org.apache.hadoop.net.DNSDomainNameResolver
hadoop.http.authentication.kerberos.keytab	${user.home}/hadoop.keytab
hadoop.http.authentication.kerberos.principal	HTTP/_HOST@LOCALHOST
hadoop.http.authentication.signature.secret.file	*********(redacted)
hadoop.http.authentication.simple.anonymous.allowed	true
hadoop.http.authentication.token.validity	*********(redacted)
hadoop.http.authentication.type	simple
hadoop.http.cross-origin.allowed-headers	X-Requested-With,Content-Type,Accept,Origin
hadoop.http.cross-origin.allowed-methods	GET,POST,HEAD
hadoop.http.cross-origin.allowed-origins	*
hadoop.http.cross-origin.enabled	false
hadoop.http.cross-origin.max-age	1800
hadoop.http.filter.initializers	org.apache.hadoop.http.lib.StaticUserWebFilter
hadoop.http.idle_timeout.ms	60000
hadoop.http.logs.enabled	true
hadoop.http.sni.host.check.enabled	false
hadoop.http.staticuser.user	dr.who
hadoop.jetty.logs.serve.aliases	true
hadoop.kerberos.keytab.login.autorenewal.enabled	false
hadoop.kerberos.kinit.command	kinit
hadoop.kerberos.min.seconds.before.relogin	60
hadoop.metrics.jvm.use-thread-mxbean	false
hadoop.prometheus.endpoint.enabled	false
hadoop.registry.jaas.context	Client
hadoop.registry.secure	false
hadoop.registry.system.acls	sasl:yarn@, sasl:mapred@, sasl:hdfs@
hadoop.registry.zk.connection.timeout.ms	15000
hadoop.registry.zk.quorum	localhost:2181
hadoop.registry.zk.retry.ceiling.ms	60000
hadoop.registry.zk.retry.interval.ms	1000
hadoop.registry.zk.retry.times	5
hadoop.registry.zk.root	/registry
hadoop.registry.zk.session.timeout.ms	60000
hadoop.rpc.protection	authentication
hadoop.rpc.socket.factory.class.default	org.apache.hadoop.net.StandardSocketFactory
hadoop.security.auth_to_local.mechanism	hadoop
hadoop.security.authentication	simple
hadoop.security.authorization	false
hadoop.security.credential.clear-text-fallback	true
hadoop.security.crypto.buffer.size	8192
hadoop.security.crypto.cipher.suite	AES/CTR/NoPadding
hadoop.security.crypto.codec.classes.aes.ctr.nopadding	org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec
hadoop.security.dns.log-slow-lookups.enabled	false
hadoop.security.dns.log-slow-lookups.threshold.ms	1000
hadoop.security.group.mapping	org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback
hadoop.security.group.mapping.ldap.connection.timeout.ms	60000
hadoop.security.group.mapping.ldap.conversion.rule	none
hadoop.security.group.mapping.ldap.directory.search.timeout	10000
hadoop.security.group.mapping.ldap.num.attempts	3
hadoop.security.group.mapping.ldap.num.attempts.before.failover	3
hadoop.security.group.mapping.ldap.posix.attr.gid.name	gidNumber
hadoop.security.group.mapping.ldap.posix.attr.uid.name	uidNumber
hadoop.security.group.mapping.ldap.read.timeout.ms	60000
hadoop.security.group.mapping.ldap.search.attr.group.name	cn
hadoop.security.group.mapping.ldap.search.attr.member	member
hadoop.security.group.mapping.ldap.search.filter.group	(objectClass=group)
hadoop.security.group.mapping.ldap.search.filter.user	(&(objectClass=user)(sAMAccountName={0}))
hadoop.security.group.mapping.ldap.search.group.hierarchy.levels	0
hadoop.security.group.mapping.ldap.ssl	false
hadoop.security.group.mapping.providers.combined	true
hadoop.security.groups.cache.background.reload	false
hadoop.security.groups.cache.background.reload.threads	3
hadoop.security.groups.cache.secs	300
hadoop.security.groups.cache.warn.after.ms	5000
hadoop.security.groups.negative-cache.secs	30
hadoop.security.groups.shell.command.timeout	0s
hadoop.security.instrumentation.requires.admin	false
hadoop.security.java.secure.random.algorithm	SHA1PRNG
hadoop.security.key.default.bitlength	128
hadoop.security.key.default.cipher	AES/CTR/NoPadding
hadoop.security.kms.client.authentication.retry-count	1
hadoop.security.kms.client.encrypted.key.cache.expiry	43200000
hadoop.security.kms.client.encrypted.key.cache.low-watermark	0.3f
hadoop.security.kms.client.encrypted.key.cache.num.refill.threads	2
hadoop.security.kms.client.encrypted.key.cache.size	500
hadoop.security.kms.client.failover.sleep.base.millis	100
hadoop.security.kms.client.failover.sleep.max.millis	2000
hadoop.security.kms.client.timeout	60
hadoop.security.random.device.file.path	/dev/urandom
hadoop.security.secure.random.impl	org.apache.hadoop.crypto.random.OpensslSecureRandom
hadoop.security.sensitive-config-keys	*********(redacted)
hadoop.security.token.service.use_ip	*********(redacted)
hadoop.security.uid.cache.secs	14400
hadoop.service.shutdown.timeout	30s
hadoop.shell.missing.defaultFs.warning	false
hadoop.shell.safely.delete.limit.num.files	100
hadoop.ssl.client.conf	ssl-client.xml
hadoop.ssl.enabled.protocols	TLSv1.2
hadoop.ssl.hostname.verifier	DEFAULT
hadoop.ssl.keystores.factory.class	org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory
hadoop.ssl.require.client.cert	false
hadoop.ssl.server.conf	ssl-server.xml
hadoop.system.tags	YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL
hadoop.tags.system	YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL
hadoop.tmp.dir	/tmp/hadoop-${user.name}
hadoop.user.group.static.mapping.overrides	dr.who=;
hadoop.util.hash.type	murmur
hadoop.workaround.non.threadsafe.getpwuid	true
hadoop.zk.acl	world:anyone:rwcda
hadoop.zk.num-retries	1000
hadoop.zk.retry-interval-ms	1000
hadoop.zk.timeout-ms	10000
io.bytes.per.checksum	512
io.compression.codec.bzip2.library	system-native
io.erasurecode.codec.rs-legacy.rawcoders	rs-legacy_java
io.erasurecode.codec.rs.rawcoders	rs_native,rs_java
io.erasurecode.codec.xor.rawcoders	xor_native,xor_java
io.file.buffer.size	65536
io.map.index.interval	128
io.map.index.skip	0
io.mapfile.bloom.error.rate	0.005
io.mapfile.bloom.size	1048576
io.seqfile.compress.blocksize	1000000
io.seqfile.local.dir	${hadoop.tmp.dir}/io/local
io.serializations	org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
io.skip.checksum.errors	false
ipc.[port_number].backoff.enable	false
ipc.[port_number].callqueue.impl	java.util.concurrent.LinkedBlockingQueue
ipc.[port_number].cost-provider.impl	org.apache.hadoop.ipc.DefaultCostProvider
ipc.[port_number].decay-scheduler.backoff.responsetime.enable	false
ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds	10s,20s,30s,40s
ipc.[port_number].decay-scheduler.decay-factor	0.5
ipc.[port_number].decay-scheduler.metrics.top.user.count	10
ipc.[port_number].decay-scheduler.period-ms	5000
ipc.[port_number].decay-scheduler.thresholds	13,25,50
ipc.[port_number].faircallqueue.multiplexer.weights	8,4,2,1
ipc.[port_number].identity-provider.impl	org.apache.hadoop.ipc.UserIdentityProvider
ipc.[port_number].scheduler.impl	org.apache.hadoop.ipc.DefaultRpcScheduler
ipc.[port_number].scheduler.priority.levels	4
ipc.[port_number].weighted-cost.handler	1
ipc.[port_number].weighted-cost.lockexclusive	100
ipc.[port_number].weighted-cost.lockfree	1
ipc.[port_number].weighted-cost.lockshared	10
ipc.[port_number].weighted-cost.response	1
ipc.client.bind.wildcard.addr	false
ipc.client.connect.max.retries	10
ipc.client.connect.max.retries.on.timeouts	45
ipc.client.connect.retry.interval	1000
ipc.client.connect.timeout	20000
ipc.client.connection.maxidletime	10000
ipc.client.fallback-to-simple-auth-allowed	false
ipc.client.idlethreshold	4000
ipc.client.kill.max	10
ipc.client.low-latency	false
ipc.client.ping	true
ipc.client.rpc-timeout.ms	0
ipc.client.tcpnodelay	true
ipc.maximum.data.length	134217728
ipc.maximum.response.length	134217728
ipc.ping.interval	60000
ipc.server.listen.queue.size	256
ipc.server.log.slow.rpc	false
ipc.server.max.connections	0
ipc.server.purge.interval	15
ipc.server.reuseaddr	true
map.sort.class	org.apache.hadoop.util.QuickSort
mapreduce.am.max-attempts	2
mapreduce.app-submission.cross-platform	false
mapreduce.client.completion.pollinterval	5000
mapreduce.client.libjars.wildcard	true
mapreduce.client.output.filter	FAILED
mapreduce.client.progressmonitor.pollinterval	1000
mapreduce.client.submit.file.replication	10
mapreduce.cluster.acls.enabled	false
mapreduce.cluster.local.dir	${hadoop.tmp.dir}/mapred/local
mapreduce.fileoutputcommitter.algorithm.version	1
mapreduce.fileoutputcommitter.task.cleanup.enabled	false
mapreduce.framework.name	local
mapreduce.ifile.readahead	true
mapreduce.ifile.readahead.bytes	4194304
mapreduce.input.fileinputformat.list-status.num-threads	1
mapreduce.input.fileinputformat.split.minsize	0
mapreduce.input.lineinputformat.linespermap	1
mapreduce.job.acl-modify-job
mapreduce.job.acl-view-job
mapreduce.job.cache.limit.max-resources	0
mapreduce.job.cache.limit.max-resources-mb	0
mapreduce.job.cache.limit.max-single-resource-mb	0
mapreduce.job.classloader	false
mapreduce.job.committer.setup.cleanup.needed	true
mapreduce.job.complete.cancel.delegation.tokens	*********(redacted)
mapreduce.job.counters.max	120
mapreduce.job.dfs.storage.capacity.kill-limit-exceed	false
mapreduce.job.emit-timeline-data	false
mapreduce.job.encrypted-intermediate-data	false
mapreduce.job.encrypted-intermediate-data-key-size-bits	128
mapreduce.job.encrypted-intermediate-data.buffer.kb	128
mapreduce.job.end-notification.max.attempts	5
mapreduce.job.end-notification.max.retry.interval	5000
mapreduce.job.end-notification.retry.attempts	0
mapreduce.job.end-notification.retry.interval	1000
mapreduce.job.finish-when-all-reducers-done	true
mapreduce.job.hdfs-servers	${fs.defaultFS}
mapreduce.job.heap.memory-mb.ratio	0.8
mapreduce.job.local-fs.single-disk-limit.bytes	-1
mapreduce.job.local-fs.single-disk-limit.check.interval-ms	5000
mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed	true
mapreduce.job.map.output.collector.class	org.apache.hadoop.mapred.MapTask$MapOutputBuffer
mapreduce.job.maps	2
mapreduce.job.max.map	-1
mapreduce.job.max.split.locations	15
mapreduce.job.maxtaskfailures.per.tracker	3
mapreduce.job.queuename	default
mapreduce.job.reduce.shuffle.consumer.plugin.class	org.apache.hadoop.mapreduce.task.reduce.Shuffle
mapreduce.job.reduce.slowstart.completedmaps	0.05
mapreduce.job.reducer.preempt.delay.sec	0
mapreduce.job.reducer.unconditional-preempt.delay.sec	300
mapreduce.job.reduces	1
mapreduce.job.running.map.limit	0
mapreduce.job.running.reduce.limit	0
mapreduce.job.sharedcache.mode	disabled
mapreduce.job.speculative.minimum-allowed-tasks	10
mapreduce.job.speculative.retry-after-no-speculate	1000
mapreduce.job.speculative.retry-after-speculate	15000
mapreduce.job.speculative.slowtaskthreshold	1.0
mapreduce.job.speculative.speculative-cap-running-tasks	0.1
mapreduce.job.speculative.speculative-cap-total-tasks	0.01
mapreduce.job.split.metainfo.maxsize	10000000
mapreduce.job.token.tracking.ids.enabled	*********(redacted)
mapreduce.job.ubertask.enable	false
mapreduce.job.ubertask.maxmaps	9
mapreduce.job.ubertask.maxreduces	1
mapreduce.jobhistory.address	0.0.0.0:10020
mapreduce.jobhistory.admin.acl	*
mapreduce.jobhistory.admin.address	0.0.0.0:10033
mapreduce.jobhistory.always-scan-user-dir	false
mapreduce.jobhistory.cleaner.enable	true
mapreduce.jobhistory.cleaner.interval-ms	86400000
mapreduce.jobhistory.client.thread-count	10
mapreduce.jobhistory.datestring.cache.size	200000
mapreduce.jobhistory.done-dir	${yarn.app.mapreduce.am.staging-dir}/history/done
mapreduce.jobhistory.http.policy	HTTP_ONLY
mapreduce.jobhistory.intermediate-done-dir	${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate
mapreduce.jobhistory.intermediate-user-done-dir.permissions	770
mapreduce.jobhistory.jhist.format	binary
mapreduce.jobhistory.joblist.cache.size	20000
mapreduce.jobhistory.jobname.limit	50
mapreduce.jobhistory.keytab	/etc/security/keytab/jhs.service.keytab
mapreduce.jobhistory.loadedjob.tasks.max	-1
mapreduce.jobhistory.loadedjobs.cache.size	5
mapreduce.jobhistory.max-age-ms	604800000
mapreduce.jobhistory.minicluster.fixed.ports	false
mapreduce.jobhistory.move.interval-ms	180000
mapreduce.jobhistory.move.thread-count	3
mapreduce.jobhistory.principal	jhs/_HOST@REALM.TLD
mapreduce.jobhistory.recovery.enable	false
mapreduce.jobhistory.recovery.store.class	org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService
mapreduce.jobhistory.recovery.store.fs.uri	${hadoop.tmp.dir}/mapred/history/recoverystore
mapreduce.jobhistory.recovery.store.leveldb.path	${hadoop.tmp.dir}/mapred/history/recoverystore
mapreduce.jobhistory.webapp.address	0.0.0.0:19888
mapreduce.jobhistory.webapp.https.address	0.0.0.0:19890
mapreduce.jobhistory.webapp.rest-csrf.custom-header	X-XSRF-Header
mapreduce.jobhistory.webapp.rest-csrf.enabled	false
mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
mapreduce.jobhistory.webapp.xfs-filter.xframe-options	SAMEORIGIN
mapreduce.jvm.system-properties-to-log	os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name
mapreduce.map.cpu.vcores	1
mapreduce.map.log.level	INFO
mapreduce.map.maxattempts	4
mapreduce.map.memory.mb	-1
mapreduce.map.output.compress	false
mapreduce.map.output.compress.codec	org.apache.hadoop.io.compress.DefaultCodec
mapreduce.map.skip.maxrecords	0
mapreduce.map.skip.proc-count.auto-incr	true
mapreduce.map.sort.spill.percent	0.80
mapreduce.map.speculative	true
mapreduce.output.fileoutputformat.compress	false
mapreduce.output.fileoutputformat.compress.codec	org.apache.hadoop.io.compress.DefaultCodec
mapreduce.output.fileoutputformat.compress.type	RECORD
mapreduce.outputcommitter.factory.scheme.s3a	org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory
mapreduce.reduce.cpu.vcores	1
mapreduce.reduce.input.buffer.percent	0.0
mapreduce.reduce.log.level	INFO
mapreduce.reduce.markreset.buffer.percent	0.0
mapreduce.reduce.maxattempts	4
mapreduce.reduce.memory.mb	-1
mapreduce.reduce.merge.inmem.threshold	1000
mapreduce.reduce.shuffle.connect.timeout	180000
mapreduce.reduce.shuffle.fetch.retry.enabled	${yarn.nodemanager.recovery.enabled}
mapreduce.reduce.shuffle.fetch.retry.interval-ms	1000
mapreduce.reduce.shuffle.fetch.retry.timeout-ms	30000
mapreduce.reduce.shuffle.input.buffer.percent	0.70
mapreduce.reduce.shuffle.memory.limit.percent	0.25
mapreduce.reduce.shuffle.merge.percent	0.66
mapreduce.reduce.shuffle.parallelcopies	5
mapreduce.reduce.shuffle.read.timeout	180000
mapreduce.reduce.shuffle.retry-delay.max.ms	60000
mapreduce.reduce.skip.maxgroups	0
mapreduce.reduce.skip.proc-count.auto-incr	true
mapreduce.reduce.speculative	true
mapreduce.shuffle.connection-keep-alive.enable	false
mapreduce.shuffle.connection-keep-alive.timeout	5
mapreduce.shuffle.listen.queue.size	128
mapreduce.shuffle.max.connections	0
mapreduce.shuffle.max.threads	0
mapreduce.shuffle.pathcache.concurrency-level	16
mapreduce.shuffle.pathcache.expire-after-access-minutes	5
mapreduce.shuffle.pathcache.max-weight	10485760
mapreduce.shuffle.port	13562
mapreduce.shuffle.ssl.enabled	false
mapreduce.shuffle.ssl.file.buffer.size	65536
mapreduce.shuffle.transfer.buffer.size	131072
mapreduce.task.combine.progress.records	10000
mapreduce.task.exit.timeout	60000
mapreduce.task.exit.timeout.check-interval-ms	20000
mapreduce.task.files.preserve.failedtasks	false
mapreduce.task.io.sort.factor	10
mapreduce.task.io.sort.mb	100
mapreduce.task.local-fs.write-limit.bytes	-1
mapreduce.task.merge.progress.records	10000
mapreduce.task.profile	false
mapreduce.task.profile.map.params	${mapreduce.task.profile.params}
mapreduce.task.profile.maps	0-2
mapreduce.task.profile.params	-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s
mapreduce.task.profile.reduce.params	${mapreduce.task.profile.params}
mapreduce.task.profile.reduces	0-2
mapreduce.task.skip.start.attempts	2
mapreduce.task.stuck.timeout-ms	600000
mapreduce.task.timeout	600000
mapreduce.task.userlog.limit.kb	0
net.topology.impl	org.apache.hadoop.net.NetworkTopology
net.topology.node.switch.mapping.impl	org.apache.hadoop.net.ScriptBasedMapping
net.topology.script.number.args	100
nfs.exports.allowed.hosts	* rw
rpc.metrics.quantile.enable	false
rpc.metrics.timeunit	MILLISECONDS
seq.io.sort.factor	100
seq.io.sort.mb	100
tfile.fs.input.buffer.size	262144
tfile.fs.output.buffer.size	262144
tfile.io.chunk.size	1048576
yarn.acl.enable	false
yarn.acl.reservation-enable	false
yarn.admin.acl	*
yarn.am.liveness-monitor.expiry-interval-ms	600000
yarn.app.attempt.diagnostics.limit.kc	64
yarn.app.mapreduce.am.command-opts	-Xmx1024m
yarn.app.mapreduce.am.container.log.backups	0
yarn.app.mapreduce.am.container.log.limit.kb	0
yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size	10
yarn.app.mapreduce.am.hard-kill-timeout-ms	10000
yarn.app.mapreduce.am.job.committer.cancel-timeout	60000
yarn.app.mapreduce.am.job.committer.commit-window	10000
yarn.app.mapreduce.am.job.task.listener.thread-count	30
yarn.app.mapreduce.am.log.level	INFO
yarn.app.mapreduce.am.resource.cpu-vcores	1
yarn.app.mapreduce.am.resource.mb	1536
yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms	1000
yarn.app.mapreduce.am.staging-dir	/tmp/hadoop-yarn/staging
yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled	false
yarn.app.mapreduce.am.webapp.https.client.auth	false
yarn.app.mapreduce.am.webapp.https.enabled	false
yarn.app.mapreduce.client-am.ipc.max-retries	3
yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts	3
yarn.app.mapreduce.client.job.max-retries	3
yarn.app.mapreduce.client.job.retry-interval	2000
yarn.app.mapreduce.client.max-retries	3
yarn.app.mapreduce.shuffle.log.backups	0
yarn.app.mapreduce.shuffle.log.limit.kb	0
yarn.app.mapreduce.shuffle.log.separate	true
yarn.app.mapreduce.task.container.log.backups	0
yarn.client.application-client-protocol.poll-interval-ms	200
yarn.client.application-client-protocol.poll-timeout-ms	-1
yarn.client.failover-no-ha-proxy-provider	org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider
yarn.client.failover-proxy-provider	org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider
yarn.client.failover-retries	0
yarn.client.failover-retries-on-socket-timeouts	0
yarn.client.load.resource-types.from-server	false
yarn.client.max-cached-nodemanagers-proxies	0
yarn.client.nodemanager-client-async.thread-pool-max-size	500
yarn.client.nodemanager-connect.max-wait-ms	180000
yarn.client.nodemanager-connect.retry-interval-ms	10000
yarn.cluster.max-application-priority	0
yarn.dispatcher.cpu-monitor.samples-per-min	60
yarn.dispatcher.drain-events.timeout	300000
yarn.dispatcher.print-events-info.threshold	5000
yarn.fail-fast	false
yarn.federation.cache-ttl.secs	300
yarn.federation.enabled	false
yarn.federation.registry.base-dir	yarnfederation/
yarn.federation.state-store.class	org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore
yarn.federation.subcluster-resolver.class	org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl
yarn.http.policy	HTTP_ONLY
yarn.intermediate-data-encryption.enable	false
yarn.ipc.rpc.class	org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC
yarn.is.minicluster	false
yarn.log-aggregation-enable	false
yarn.log-aggregation-status.time-out.ms	600000
yarn.log-aggregation.debug.filesize	104857600
yarn.log-aggregation.file-controller.TFile.class	org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController
yarn.log-aggregation.file-formats	TFile
yarn.log-aggregation.retain-check-interval-seconds	-1
yarn.log-aggregation.retain-seconds	-1
yarn.minicluster.control-resource-monitoring	false
yarn.minicluster.fixed.ports	false
yarn.minicluster.use-rpc	false
yarn.minicluster.yarn.nodemanager.resource.memory-mb	4096
yarn.nm.liveness-monitor.expiry-interval-ms	600000
yarn.node-attribute.fs-store.impl.class	org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore
yarn.node-labels.configuration-type	centralized
yarn.node-labels.enabled	false
yarn.node-labels.fs-store.impl.class	org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore
yarn.nodemanager.address	${yarn.nodemanager.hostname}:0
yarn.nodemanager.admin-env	MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX
yarn.nodemanager.amrmproxy.address	0.0.0.0:8049
yarn.nodemanager.amrmproxy.client.thread-count	25
yarn.nodemanager.amrmproxy.enabled	false
yarn.nodemanager.amrmproxy.ha.enable	false
yarn.nodemanager.amrmproxy.interceptor-class.pipeline	org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor
yarn.nodemanager.aux-services.manifest.enabled	false
yarn.nodemanager.aux-services.manifest.reload-ms	0
yarn.nodemanager.aux-services.mapreduce_shuffle.class	org.apache.hadoop.mapred.ShuffleHandler
yarn.nodemanager.collector-service.address	${yarn.nodemanager.hostname}:8048
yarn.nodemanager.collector-service.thread-count	5
yarn.nodemanager.container-diagnostics-maximum-size	10000
yarn.nodemanager.container-executor.class	org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor
yarn.nodemanager.container-executor.exit-code-file.timeout-ms	2000
yarn.nodemanager.container-localizer.java.opts	-Xmx256m
yarn.nodemanager.container-localizer.log.level	INFO
yarn.nodemanager.container-log-monitor.dir-size-limit-bytes	1000000000
yarn.nodemanager.container-log-monitor.enable	false
yarn.nodemanager.container-log-monitor.interval-ms	60000
yarn.nodemanager.container-log-monitor.total-size-limit-bytes	10000000000
yarn.nodemanager.container-manager.thread-count	20
yarn.nodemanager.container-metrics.enable	true
yarn.nodemanager.container-metrics.period-ms	-1
yarn.nodemanager.container-metrics.unregister-delay-ms	10000
yarn.nodemanager.container-monitor.enabled	true
yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled	false
yarn.nodemanager.container-retry-minimum-interval-ms	1000
yarn.nodemanager.container.stderr.pattern	{stderr,STDERR}
yarn.nodemanager.container.stderr.tail.bytes	4096
yarn.nodemanager.containers-launcher.class	org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher
yarn.nodemanager.default-container-executor.log-dirs.permissions	710
yarn.nodemanager.delete.debug-delay-sec	0
yarn.nodemanager.delete.thread-count	4
yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled	true
yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled	true
yarn.nodemanager.disk-health-checker.enable	true
yarn.nodemanager.disk-health-checker.interval-ms	120000
yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage	90.0
yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb	0
yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb	0
yarn.nodemanager.disk-health-checker.min-healthy-disks	0.25
yarn.nodemanager.disk-validator	basic
yarn.nodemanager.distributed-scheduling.enabled	false
yarn.nodemanager.elastic-memory-control.enabled	false
yarn.nodemanager.elastic-memory-control.oom-handler	org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler
yarn.nodemanager.elastic-memory-control.timeout-sec	5
yarn.nodemanager.emit-container-events	true
yarn.nodemanager.env-whitelist	JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ
yarn.nodemanager.health-checker.interval-ms	600000
yarn.nodemanager.health-checker.run-before-startup	false
yarn.nodemanager.health-checker.scripts	script
yarn.nodemanager.health-checker.timeout-ms	1200000
yarn.nodemanager.hostname	0.0.0.0
yarn.nodemanager.keytab	/etc/krb5.keytab
yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms	20
yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms	1000
yarn.nodemanager.linux-container-executor.cgroups.hierarchy	/hadoop-yarn
yarn.nodemanager.linux-container-executor.cgroups.mount	false
yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage	false
yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users	true
yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user	nobody
yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern	^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$
yarn.nodemanager.linux-container-executor.resources-handler.class	org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler
yarn.nodemanager.local-cache.max-files-per-directory	8192
yarn.nodemanager.local-dirs	${hadoop.tmp.dir}/nm-local-dir
yarn.nodemanager.localizer.address	${yarn.nodemanager.hostname}:8040
yarn.nodemanager.localizer.cache.cleanup.interval-ms	600000
yarn.nodemanager.localizer.cache.target-size-mb	10240
yarn.nodemanager.localizer.client.thread-count	5
yarn.nodemanager.localizer.fetch.thread-count	4
yarn.nodemanager.log-aggregation.compression-type	none
yarn.nodemanager.log-aggregation.num-log-files-per-app	30
yarn.nodemanager.log-aggregation.policy.class	org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy
yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds	-1
yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min	3600
yarn.nodemanager.log-container-debug-info.enabled	true
yarn.nodemanager.log-dirs	${yarn.log.dir}/userlogs
yarn.nodemanager.log.deletion-threads-count	4
yarn.nodemanager.log.retain-seconds	10800
yarn.nodemanager.logaggregation.threadpool-size-max	100
yarn.nodemanager.node-attributes.provider.fetch-interval-ms	600000
yarn.nodemanager.node-attributes.provider.fetch-timeout-ms	1200000
yarn.nodemanager.node-attributes.resync-interval-ms	120000
yarn.nodemanager.node-labels.provider.fetch-interval-ms	600000
yarn.nodemanager.node-labels.provider.fetch-timeout-ms	1200000
yarn.nodemanager.node-labels.resync-interval-ms	120000
yarn.nodemanager.numa-awareness.enabled	false
yarn.nodemanager.numa-awareness.numactl.cmd	/usr/bin/numactl
yarn.nodemanager.numa-awareness.read-topology	false
yarn.nodemanager.opportunistic-containers-max-queue-length	0
yarn.nodemanager.opportunistic-containers-use-pause-for-preemption	false
yarn.nodemanager.pluggable-device-framework.enabled	false
yarn.nodemanager.pmem-check-enabled	true
yarn.nodemanager.process-kill-wait.ms	5000
yarn.nodemanager.recovery.compaction-interval-secs	3600
yarn.nodemanager.recovery.dir	${hadoop.tmp.dir}/yarn-nm-recovery
yarn.nodemanager.recovery.enabled	false
yarn.nodemanager.recovery.supervised	false
yarn.nodemanager.remote-app-log-dir	/tmp/logs
yarn.nodemanager.remote-app-log-dir-include-older	true
yarn.nodemanager.remote-app-log-dir-suffix	logs
yarn.nodemanager.resource-monitor.interval-ms	3000
yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices	auto
yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class	org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin
yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices	auto
yarn.nodemanager.resource-plugins.gpu.docker-plugin	nvidia-docker-v1
yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint	http://localhost:3476/v1.0/docker/cli
yarn.nodemanager.resource.count-logical-processors-as-cores	false
yarn.nodemanager.resource.cpu-vcores	-1
yarn.nodemanager.resource.detect-hardware-capabilities	false
yarn.nodemanager.resource.memory-mb	-1
yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage	90.0
yarn.nodemanager.resource.memory.cgroups.swappiness	0
yarn.nodemanager.resource.memory.enabled	false
yarn.nodemanager.resource.memory.enforced	true
yarn.nodemanager.resource.pcores-vcores-multiplier	1.0
yarn.nodemanager.resource.percentage-physical-cpu-limit	100
yarn.nodemanager.resource.system-reserved-memory-mb	-1
yarn.nodemanager.resourcemanager.minimum.version	NONE
yarn.nodemanager.runtime.linux.allowed-runtimes	default
yarn.nodemanager.runtime.linux.docker.allowed-container-networks	host,none,bridge
yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes	runc
yarn.nodemanager.runtime.linux.docker.capabilities	CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
yarn.nodemanager.runtime.linux.docker.default-container-network	host
yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed	false
yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed	true
yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed	false
yarn.nodemanager.runtime.linux.docker.image-update	false
yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed	false
yarn.nodemanager.runtime.linux.docker.stop.grace-period	10
yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold	1
yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold	1
yarn.nodemanager.runtime.linux.runc.allowed-container-networks	host,none,bridge
yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes	runc
yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size	500
yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs	360
yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed	false
yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin	org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin
yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs	60
yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file	/runc-root/image-tag-to-hash
yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache	10
yarn.nodemanager.runtime.linux.runc.image-toplevel-dir	/runc-root
yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs	600
yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep	100
yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin	org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin
yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed	false
yarn.nodemanager.runtime.linux.sandbox-mode	disabled
yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions	read
yarn.nodemanager.sleep-delay-before-sigkill.ms	250
yarn.nodemanager.vmem-check-enabled	true
yarn.nodemanager.vmem-pmem-ratio	2.1
yarn.nodemanager.webapp.address	${yarn.nodemanager.hostname}:8042
yarn.nodemanager.webapp.cross-origin.enabled	false
yarn.nodemanager.webapp.https.address	0.0.0.0:8044
yarn.nodemanager.webapp.rest-csrf.custom-header	X-XSRF-Header
yarn.nodemanager.webapp.rest-csrf.enabled	false
yarn.nodemanager.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
yarn.nodemanager.webapp.xfs-filter.xframe-options	SAMEORIGIN
yarn.nodemanager.windows-container.cpu-limit.enabled	false
yarn.nodemanager.windows-container.memory-limit.enabled	false
yarn.registry.class	org.apache.hadoop.registry.client.impl.FSRegistryOperationsService
yarn.resourcemanager.activities-manager.app-activities.max-queue-length	100
yarn.resourcemanager.activities-manager.app-activities.ttl-ms	600000
yarn.resourcemanager.activities-manager.cleanup-interval-ms	5000
yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms	600000
yarn.resourcemanager.address	${yarn.resourcemanager.hostname}:8032
yarn.resourcemanager.admin.address	${yarn.resourcemanager.hostname}:8033
yarn.resourcemanager.admin.client.thread-count	1
yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs	*********(redacted)
yarn.resourcemanager.am.max-attempts	2
yarn.resourcemanager.amlauncher.thread-count	50
yarn.resourcemanager.application-https.policy	NONE
yarn.resourcemanager.application-tag-based-placement.enable	false
yarn.resourcemanager.application-timeouts.monitor.interval-ms	3000
yarn.resourcemanager.application.max-tag.length	100
yarn.resourcemanager.application.max-tags	10
yarn.resourcemanager.auto-update.containers	false
yarn.resourcemanager.client.thread-count	50
yarn.resourcemanager.configuration.file-system-based-store	/yarn/conf
yarn.resourcemanager.configuration.provider-class	org.apache.hadoop.yarn.LocalConfigurationProvider
yarn.resourcemanager.connect.max-wait.ms	900000
yarn.resourcemanager.connect.retry-interval.ms	30000
yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs	*********(redacted)
yarn.resourcemanager.container.liveness-monitor.interval-ms	600000
yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs	20
yarn.resourcemanager.delayed.delegation-token.removal-interval-ms	*********(redacted)
yarn.resourcemanager.delegation-token-renewer.thread-count	*********(redacted)
yarn.resourcemanager.delegation-token-renewer.thread-retry-interval	*********(redacted)
yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts	*********(redacted)
yarn.resourcemanager.delegation-token-renewer.thread-timeout	*********(redacted)
yarn.resourcemanager.delegation-token.always-cancel	*********(redacted)
yarn.resourcemanager.delegation-token.max-conf-size-bytes	*********(redacted)
yarn.resourcemanager.delegation.key.update-interval	86400000
yarn.resourcemanager.delegation.token.max-lifetime	*********(redacted)
yarn.resourcemanager.delegation.token.renew-interval	*********(redacted)
yarn.resourcemanager.epoch.range	0
yarn.resourcemanager.fail-fast	${yarn.fail-fast}
yarn.resourcemanager.fs.state-store.num-retries	0
yarn.resourcemanager.fs.state-store.retry-interval-ms	1000
yarn.resourcemanager.fs.state-store.uri	${hadoop.tmp.dir}/yarn/system/rmstore
yarn.resourcemanager.ha.automatic-failover.embedded	true
yarn.resourcemanager.ha.automatic-failover.enabled	true
yarn.resourcemanager.ha.automatic-failover.zk-base-path	/yarn-leader-election
yarn.resourcemanager.ha.enabled	false
yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size	10
yarn.resourcemanager.hostname	0.0.0.0
yarn.resourcemanager.keytab	/etc/krb5.keytab
yarn.resourcemanager.leveldb-state-store.compaction-interval-secs	3600
yarn.resourcemanager.leveldb-state-store.path	${hadoop.tmp.dir}/yarn/system/rmstore
yarn.resourcemanager.max-completed-applications	1000
yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory	10
yarn.resourcemanager.metrics.runtime.buckets	60,300,1440
yarn.resourcemanager.nm-container-queuing.load-comparator	QUEUE_LENGTH
yarn.resourcemanager.nm-container-queuing.max-queue-length	15
yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms	100
yarn.resourcemanager.nm-container-queuing.min-queue-length	5
yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms	10
yarn.resourcemanager.nm-container-queuing.queue-limit-stdev	1.0f
yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms	1000
yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs	*********(redacted)
yarn.resourcemanager.node-ip-cache.expiry-interval-secs	-1
yarn.resourcemanager.node-labels.provider.fetch-interval-ms	1800000
yarn.resourcemanager.node-removal-untracked.timeout-ms	60000
yarn.resourcemanager.nodemanager-connect-retries	10
yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs	3600
yarn.resourcemanager.nodemanager.minimum.version	NONE
yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms	1000
yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms	1000
yarn.resourcemanager.nodemanagers.heartbeat-interval-ms	1000
yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable	false
yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor	1.0
yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor	1.0
yarn.resourcemanager.opportunistic-container-allocation.enabled	false
yarn.resourcemanager.opportunistic-container-allocation.nodes-used	10
yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat	-1
yarn.resourcemanager.placement-constraints.algorithm.class	org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm
yarn.resourcemanager.placement-constraints.algorithm.iterator	SERIAL
yarn.resourcemanager.placement-constraints.algorithm.pool-size	1
yarn.resourcemanager.placement-constraints.handler	disabled
yarn.resourcemanager.placement-constraints.retry-attempts	3
yarn.resourcemanager.placement-constraints.scheduler.pool-size	1
yarn.resourcemanager.proxy-user-privileges.enabled	false
yarn.resourcemanager.proxy.connection.timeout	60000
yarn.resourcemanager.proxy.timeout.enabled	true
yarn.resourcemanager.recovery.enabled	false
yarn.resourcemanager.reservation-system.enable	false
yarn.resourcemanager.reservation-system.planfollower.time-step	1000
yarn.resourcemanager.resource-profiles.enabled	false
yarn.resourcemanager.resource-profiles.source-file	resource-profiles.json
yarn.resourcemanager.resource-tracker.address	${yarn.resourcemanager.hostname}:8031
yarn.resourcemanager.resource-tracker.client.thread-count	50
yarn.resourcemanager.resource-tracker.nm.ip-hostname-check	false
yarn.resourcemanager.rm.container-allocation.expiry-interval-ms	600000
yarn.resourcemanager.scheduler.address	${yarn.resourcemanager.hostname}:8030
yarn.resourcemanager.scheduler.class	org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
yarn.resourcemanager.scheduler.client.thread-count	50
yarn.resourcemanager.scheduler.monitor.enable	false
yarn.resourcemanager.scheduler.monitor.policies	org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy
yarn.resourcemanager.state-store.max-completed-applications	${yarn.resourcemanager.max-completed-applications}
yarn.resourcemanager.store.class	org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
yarn.resourcemanager.submission-preprocessor.enabled	false
yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms	60000
yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size	10
yarn.resourcemanager.system-metrics-publisher.enabled	false
yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size	1000
yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch	false
yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds	60
yarn.resourcemanager.webapp.address	${yarn.resourcemanager.hostname}:8088
yarn.resourcemanager.webapp.cross-origin.enabled	false
yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled	*********(redacted)
yarn.resourcemanager.webapp.https.address	${yarn.resourcemanager.hostname}:8090
yarn.resourcemanager.webapp.rest-csrf.custom-header	X-XSRF-Header
yarn.resourcemanager.webapp.rest-csrf.enabled	false
yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
yarn.resourcemanager.webapp.ui-actions.enabled	true
yarn.resourcemanager.webapp.xfs-filter.xframe-options	SAMEORIGIN
yarn.resourcemanager.work-preserving-recovery.enabled	true
yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms	10000
yarn.resourcemanager.zk-appid-node.split-index	0
yarn.resourcemanager.zk-delegation-token-node.split-index	*********(redacted)
yarn.resourcemanager.zk-max-znode-size.bytes	1048576
yarn.resourcemanager.zk-state-store.parent-path	/rmstore
yarn.rm.system-metrics-publisher.emit-container-events	false
yarn.router.clientrm.interceptor-class.pipeline	org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor
yarn.router.interceptor.user.threadpool-size	5
yarn.router.pipeline.cache-max-size	25
yarn.router.rmadmin.interceptor-class.pipeline	org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor
yarn.router.webapp.address	0.0.0.0:8089
yarn.router.webapp.https.address	0.0.0.0:8091
yarn.router.webapp.interceptor-class.pipeline	org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST
yarn.scheduler.configuration.fs.path	file://${hadoop.tmp.dir}/yarn/system/schedconf
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs	86400
yarn.scheduler.configuration.leveldb-store.path	${hadoop.tmp.dir}/yarn/system/confstore
yarn.scheduler.configuration.max.version	100
yarn.scheduler.configuration.mutation.acl-policy.class	org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
yarn.scheduler.configuration.store.class	file
yarn.scheduler.configuration.store.max-logs	1000
yarn.scheduler.configuration.zk-store.parent-path	/confstore
yarn.scheduler.include-port-in-node-name	false
yarn.scheduler.maximum-allocation-mb	8192
yarn.scheduler.maximum-allocation-vcores	4
yarn.scheduler.minimum-allocation-mb	1024
yarn.scheduler.minimum-allocation-vcores	1
yarn.scheduler.queue-placement-rules	user-group
yarn.sharedcache.admin.address	0.0.0.0:8047
yarn.sharedcache.admin.thread-count	1
yarn.sharedcache.app-checker.class	org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker
yarn.sharedcache.checksum.algo.impl	org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl
yarn.sharedcache.cleaner.initial-delay-mins	10
yarn.sharedcache.cleaner.period-mins	1440
yarn.sharedcache.cleaner.resource-sleep-ms	0
yarn.sharedcache.client-server.address	0.0.0.0:8045
yarn.sharedcache.client-server.thread-count	50
yarn.sharedcache.enabled	false
yarn.sharedcache.nested-level	3
yarn.sharedcache.nm.uploader.replication.factor	10
yarn.sharedcache.nm.uploader.thread-count	20
yarn.sharedcache.root-dir	/sharedcache
yarn.sharedcache.store.class	org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore
yarn.sharedcache.store.in-memory.check-period-mins	720
yarn.sharedcache.store.in-memory.initial-delay-mins	10
yarn.sharedcache.store.in-memory.staleness-period-mins	10080
yarn.sharedcache.uploader.server.address	0.0.0.0:8046
yarn.sharedcache.uploader.server.thread-count	50
yarn.sharedcache.webapp.address	0.0.0.0:8788
yarn.system-metrics-publisher.enabled	false
yarn.timeline-service.address	${yarn.timeline-service.hostname}:10200
yarn.timeline-service.app-aggregation-interval-secs	15
yarn.timeline-service.app-collector.linger-period.ms	60000
yarn.timeline-service.client.best-effort	false
yarn.timeline-service.client.drain-entities.timeout.ms	2000
yarn.timeline-service.client.fd-clean-interval-secs	60
yarn.timeline-service.client.fd-flush-interval-secs	10
yarn.timeline-service.client.fd-retain-secs	300
yarn.timeline-service.client.internal-timers-ttl-secs	420
yarn.timeline-service.client.max-retries	30
yarn.timeline-service.client.retry-interval-ms	1000
yarn.timeline-service.enabled	false
yarn.timeline-service.entity-group-fs-store.active-dir	/tmp/entity-file-history/active
yarn.timeline-service.entity-group-fs-store.app-cache-size	10
yarn.timeline-service.entity-group-fs-store.cache-store-class	org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore
yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds	3600
yarn.timeline-service.entity-group-fs-store.done-dir	/tmp/entity-file-history/done/
yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size	10485760
yarn.timeline-service.entity-group-fs-store.retain-seconds	604800
yarn.timeline-service.entity-group-fs-store.scan-interval-seconds	60
yarn.timeline-service.entity-group-fs-store.summary-store	org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
yarn.timeline-service.entity-group-fs-store.with-user-dir	false
yarn.timeline-service.flowname.max-size	0
yarn.timeline-service.generic-application-history.max-applications	10000
yarn.timeline-service.handler-thread-count	10
yarn.timeline-service.hbase-schema.prefix	prod.
yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds	259200000
yarn.timeline-service.hbase.coprocessor.jar.hdfs.location	/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar
yarn.timeline-service.hostname	0.0.0.0
yarn.timeline-service.http-authentication.simple.anonymous.allowed	true
yarn.timeline-service.http-authentication.type	simple
yarn.timeline-service.http-cross-origin.enabled	false
yarn.timeline-service.keytab	/etc/krb5.keytab
yarn.timeline-service.leveldb-state-store.path	${hadoop.tmp.dir}/yarn/timeline
yarn.timeline-service.leveldb-timeline-store.path	${hadoop.tmp.dir}/yarn/timeline
yarn.timeline-service.leveldb-timeline-store.read-cache-size	104857600
yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size	10000
yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size	10000
yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms	300000
yarn.timeline-service.reader.class	org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl
yarn.timeline-service.reader.webapp.address	${yarn.timeline-service.webapp.address}
yarn.timeline-service.reader.webapp.https.address	${yarn.timeline-service.webapp.https.address}
yarn.timeline-service.recovery.enabled	false
yarn.timeline-service.state-store-class	org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore
yarn.timeline-service.store-class	org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
yarn.timeline-service.timeline-client.number-of-async-entities-to-merge	10
yarn.timeline-service.ttl-enable	true
yarn.timeline-service.ttl-ms	604800000
yarn.timeline-service.version	1.0f
yarn.timeline-service.webapp.address	${yarn.timeline-service.hostname}:8188
yarn.timeline-service.webapp.https.address	${yarn.timeline-service.hostname}:8190
yarn.timeline-service.webapp.rest-csrf.custom-header	X-XSRF-Header
yarn.timeline-service.webapp.rest-csrf.enabled	false
yarn.timeline-service.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
yarn.timeline-service.webapp.xfs-filter.xframe-options	SAMEORIGIN
yarn.timeline-service.writer.async.queue.capacity	100
yarn.timeline-service.writer.class	org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl
yarn.timeline-service.writer.flush-interval-seconds	60
yarn.webapp.api-service.enable	false
yarn.webapp.enable-rest-app-submissions	true
yarn.webapp.filter-entity-list-by-user	false
yarn.webapp.filter-invalid-xml-chars	false
yarn.webapp.ui2.enable	false
yarn.webapp.xfs-filter.enabled	true
yarn.workflow-id.tag-prefix	workflowid:

System Properties

Name	Value
SPARK_SUBMIT	true
com.zaxxer.hikari.pool_number	1
file.encoding	UTF-8
file.separator	/
java.class.version	61.0
java.home	/opt/java/openjdk
java.io.tmpdir	/tmp
java.library.path	/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
java.runtime.name	OpenJDK Runtime Environment
java.runtime.version	17.0.12+7
java.specification.name	Java Platform API Specification
java.specification.vendor	Oracle Corporation
java.specification.version	17
java.vendor	Eclipse Adoptium
java.vendor.url	https://adoptium.net/
java.vendor.url.bug	https://github.com/adoptium/adoptium-support/issues
java.vendor.version	Temurin-17.0.12+7
java.version	17.0.12
java.version.date	2024-07-16
java.vm.compressedOopsMode	32-bit
java.vm.info	mixed mode, sharing
java.vm.name	OpenJDK 64-Bit Server VM
java.vm.specification.name	Java Virtual Machine Specification
java.vm.specification.vendor	Oracle Corporation
java.vm.specification.version	17
java.vm.vendor	Eclipse Adoptium
java.vm.version	17.0.12+7
jdk.debug	release
jdk.reflect.useDirectMethodHandle	false
jetty.git.hash	abdcda73818a1a2c705da276edb0bf6581e7997e
line.separator
native.encoding	UTF-8
os.arch	amd64
os.name	Linux
os.version	6.10.4-1-ec2
path.separator	:
sun.arch.data.model	64
sun.boot.library.path	/opt/java/openjdk/lib
sun.cpu.endian	little
sun.io.unicode.encoding	UnicodeLittle
sun.java.command	org.apache.spark.deploy.SparkSubmit data_pipeline.jar
sun.java.launcher	SUN_STANDARD
sun.jnu.encoding	UTF-8
sun.management.compiler	HotSpot 64-Bit Tiered Compilers
user.country	US
user.dir	/opt/spark
user.home	/home/spark
user.language	en
user.name	spark
user.timezone	Etc/UTC

Metrics Properties

Name	Value
*.sink.servlet.class	org.apache.spark.metrics.sink.MetricsServlet
*.sink.servlet.path	/metrics/json
applications.sink.servlet.path	/metrics/applications/json
master.sink.servlet.path	/metrics/master/json

Classpath Entries

Resource	Source
/opt/spark/conf	System Classpath
/opt/spark/jars/HikariCP-5.1.0.jar	System Classpath
/opt/spark/jars/RoaringBitmap-0.9.45.jar	System Classpath
/opt/spark/jars/activation-1.1.1.jar	System Classpath
/opt/spark/jars/aircompressor-0.25.jar	System Classpath
/opt/spark/jars/annotations-17.0.0.jar	System Classpath
/opt/spark/jars/annotations-2.23.3.jar	System Classpath
/opt/spark/jars/ant-1.10.12.jar	System Classpath
/opt/spark/jars/ant-launcher-1.10.12.jar	System Classpath
/opt/spark/jars/antlr4-runtime-4.9.3.jar	System Classpath
/opt/spark/jars/aopalliance-repackaged-2.6.1.jar	System Classpath
/opt/spark/jars/apache-client-2.23.3.jar	System Classpath
/opt/spark/jars/arrow-format-12.0.1.jar	System Classpath
/opt/spark/jars/arrow-memory-core-12.0.1.jar	System Classpath
/opt/spark/jars/arrow-memory-netty-12.0.1.jar	System Classpath
/opt/spark/jars/arrow-vector-12.0.1.jar	System Classpath
/opt/spark/jars/asm-9.6.jar	System Classpath
/opt/spark/jars/asm-commons-9.6.jar	System Classpath
/opt/spark/jars/asm-tree-9.6.jar	System Classpath
/opt/spark/jars/audience-annotations-0.13.0.jar	System Classpath
/opt/spark/jars/auth-2.23.3.jar	System Classpath
/opt/spark/jars/avro-1.11.2.jar	System Classpath
/opt/spark/jars/avro-ipc-1.11.2.jar	System Classpath
/opt/spark/jars/avro-mapred-1.11.2.jar	System Classpath
/opt/spark/jars/aws-core-2.23.3.jar	System Classpath
/opt/spark/jars/aws-java-sdk-core-1.12.638.jar	System Classpath
/opt/spark/jars/aws-java-sdk-sts-1.12.638.jar	System Classpath
/opt/spark/jars/aws-json-protocol-2.23.3.jar	System Classpath
/opt/spark/jars/aws-msk-iam-auth-2.0.3.jar	System Classpath
/opt/spark/jars/aws-query-protocol-2.23.3.jar	System Classpath
/opt/spark/jars/checksums-2.23.3.jar	System Classpath
/opt/spark/jars/checksums-spi-2.23.3.jar	System Classpath
/opt/spark/jars/chill-java-0.10.0.jar	System Classpath
/opt/spark/jars/chill_2.12-0.10.0.jar	System Classpath
/opt/spark/jars/clickhouse-client-0.6.1.jar	System Classpath
/opt/spark/jars/clickhouse-data-0.6.1.jar	System Classpath
/opt/spark/jars/clickhouse-jdbc-0.6.1.jar	System Classpath
/opt/spark/jars/commons-codec-1.16.0.jar	System Classpath
/opt/spark/jars/commons-collections-3.2.2.jar	System Classpath
/opt/spark/jars/commons-collections4-4.4.jar	System Classpath
/opt/spark/jars/commons-compiler-3.1.9.jar	System Classpath
/opt/spark/jars/commons-compress-1.21.jar	System Classpath
/opt/spark/jars/commons-compress-1.23.0.jar	System Classpath
/opt/spark/jars/commons-crypto-1.1.0.jar	System Classpath
/opt/spark/jars/commons-io-2.13.0.jar	System Classpath
/opt/spark/jars/commons-lang3-3.12.0.jar	System Classpath
/opt/spark/jars/commons-lang3-3.8.1.jar	System Classpath
/opt/spark/jars/commons-logging-1.2.jar	System Classpath
/opt/spark/jars/commons-math3-3.6.1.jar	System Classpath
/opt/spark/jars/commons-pool2-2.11.1.jar	System Classpath
/opt/spark/jars/commons-text-1.10.0.jar	System Classpath
/opt/spark/jars/compiler-bridge_2.12-1.9.5-sources.jar	System Classpath
/opt/spark/jars/compiler-interface-1.9.5.jar	System Classpath
/opt/spark/jars/compress-lzf-1.1.2.jar	System Classpath
/opt/spark/jars/curator-client-2.13.0.jar	System Classpath
/opt/spark/jars/curator-framework-2.13.0.jar	System Classpath
/opt/spark/jars/curator-recipes-2.13.0.jar	System Classpath
/opt/spark/jars/datasketches-java-3.3.0.jar	System Classpath
/opt/spark/jars/datasketches-memory-2.1.0.jar	System Classpath
/opt/spark/jars/endpoints-spi-2.23.3.jar	System Classpath
/opt/spark/jars/eventstream-1.0.1.jar	System Classpath
/opt/spark/jars/flatbuffers-java-1.12.0.jar	System Classpath
/opt/spark/jars/gson-2.10.1.jar	System Classpath
/opt/spark/jars/guava-16.0.1.jar	System Classpath
/opt/spark/jars/hadoop-client-api-3.3.4.jar	System Classpath
/opt/spark/jars/hadoop-client-runtime-3.3.4.jar	System Classpath
/opt/spark/jars/hive-storage-api-2.8.1.jar	System Classpath
/opt/spark/jars/hk2-api-2.6.1.jar	System Classpath
/opt/spark/jars/hk2-locator-2.6.1.jar	System Classpath
/opt/spark/jars/hk2-utils-2.6.1.jar	System Classpath
/opt/spark/jars/http-auth-2.23.3.jar	System Classpath
/opt/spark/jars/http-auth-aws-2.23.3.jar	System Classpath
/opt/spark/jars/http-auth-spi-2.23.3.jar	System Classpath
/opt/spark/jars/http-client-spi-2.23.3.jar	System Classpath
/opt/spark/jars/httpclient-4.5.13.jar	System Classpath
/opt/spark/jars/httpclient5-5.2.3.jar	System Classpath
/opt/spark/jars/httpcore-4.4.13.jar	System Classpath
/opt/spark/jars/httpcore5-5.2.4.jar	System Classpath
/opt/spark/jars/httpcore5-h2-5.2.4.jar	System Classpath
/opt/spark/jars/identity-spi-2.23.3.jar	System Classpath
/opt/spark/jars/ivy-2.5.1.jar	System Classpath
/opt/spark/jars/jackson-annotations-2.15.2.jar	System Classpath
/opt/spark/jars/jackson-core-2.15.2.jar	System Classpath
/opt/spark/jars/jackson-databind-2.15.2.jar	System Classpath
/opt/spark/jars/jackson-dataformat-cbor-2.12.6.jar	System Classpath
/opt/spark/jars/jackson-datatype-jsr310-2.15.1.jar	System Classpath
/opt/spark/jars/jackson-module-scala_2.12-2.15.2.jar	System Classpath
/opt/spark/jars/jakarta.annotation-api-1.3.5.jar	System Classpath
/opt/spark/jars/jakarta.inject-2.6.1.jar	System Classpath
/opt/spark/jars/jakarta.servlet-api-4.0.3.jar	System Classpath
/opt/spark/jars/jakarta.validation-api-2.0.2.jar	System Classpath
/opt/spark/jars/jakarta.ws.rs-api-2.1.6.jar	System Classpath
/opt/spark/jars/janino-3.1.9.jar	System Classpath
/opt/spark/jars/jarjar-1.14.0.jar	System Classpath
/opt/spark/jars/jarjar-abrams-core_2.12-1.14.0.jar	System Classpath
/opt/spark/jars/javassist-3.29.2-GA.jar	System Classpath
/opt/spark/jars/jcl-over-slf4j-2.0.7.jar	System Classpath
/opt/spark/jars/jersey-client-2.40.jar	System Classpath
/opt/spark/jars/jersey-common-2.40.jar	System Classpath
/opt/spark/jars/jersey-container-servlet-2.40.jar	System Classpath
/opt/spark/jars/jersey-container-servlet-core-2.40.jar	System Classpath
/opt/spark/jars/jersey-hk2-2.40.jar	System Classpath
/opt/spark/jars/jersey-server-2.40.jar	System Classpath
/opt/spark/jars/jline-2.14.6.jar	System Classpath
/opt/spark/jars/jmespath-java-1.12.638.jar	System Classpath
/opt/spark/jars/joda-time-2.12.5.jar	System Classpath
/opt/spark/jars/json-utils-2.23.3.jar	System Classpath
/opt/spark/jars/json4s-ast_2.12-3.7.0-M11.jar	System Classpath
/opt/spark/jars/json4s-core_2.12-3.7.0-M11.jar	System Classpath
/opt/spark/jars/json4s-jackson_2.12-3.7.0-M11.jar	System Classpath
/opt/spark/jars/json4s-scalap_2.12-3.7.0-M11.jar	System Classpath
/opt/spark/jars/jsr305-3.0.2.jar	System Classpath
/opt/spark/jars/jul-to-slf4j-2.0.7.jar	System Classpath
/opt/spark/jars/kafka-clients-3.6.1.jar	System Classpath
/opt/spark/jars/kryo-shaded-4.0.2.jar	System Classpath
/opt/spark/jars/leveldbjni-all-1.8.jar	System Classpath
/opt/spark/jars/log4j-1.2-api-2.20.0.jar	System Classpath
/opt/spark/jars/log4j-api-2.22.1.jar	System Classpath
/opt/spark/jars/log4j-api-scala_2.12-13.0.0.jar	System Classpath
/opt/spark/jars/log4j-core-2.22.1.jar	System Classpath
/opt/spark/jars/log4j-slf4j2-impl-2.20.0.jar	System Classpath
/opt/spark/jars/lz4-java-1.8.0.jar	System Classpath
/opt/spark/jars/metrics-core-4.2.19.jar	System Classpath
/opt/spark/jars/metrics-graphite-4.2.19.jar	System Classpath
/opt/spark/jars/metrics-jmx-4.2.19.jar	System Classpath
/opt/spark/jars/metrics-json-4.2.19.jar	System Classpath
/opt/spark/jars/metrics-jvm-4.2.19.jar	System Classpath
/opt/spark/jars/metrics-spi-2.23.3.jar	System Classpath
/opt/spark/jars/minlog-1.3.0.jar	System Classpath
/opt/spark/jars/mysql-connector-j-8.3.0.jar	System Classpath
/opt/spark/jars/netty-all-4.1.96.Final.jar	System Classpath
/opt/spark/jars/netty-buffer-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-codec-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-codec-http-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-codec-http2-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-codec-socks-4.1.96.Final.jar	System Classpath
/opt/spark/jars/netty-common-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-handler-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-handler-proxy-4.1.96.Final.jar	System Classpath
/opt/spark/jars/netty-nio-client-2.23.3.jar	System Classpath
/opt/spark/jars/netty-resolver-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-transport-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-transport-classes-epoll-4.1.100.Final.jar	System Classpath
/opt/spark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar	System Classpath
/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar	System Classpath
/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar	System Classpath
/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final.jar	System Classpath
/opt/spark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar	System Classpath
/opt/spark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar	System Classpath
/opt/spark/jars/netty-transport-native-unix-common-4.1.100.Final.jar	System Classpath
/opt/spark/jars/objenesis-2.5.1.jar	System Classpath
/opt/spark/jars/orc-core-1.9.1-shaded-protobuf.jar	System Classpath
/opt/spark/jars/orc-mapreduce-1.9.1-shaded-protobuf.jar	System Classpath
/opt/spark/jars/orc-shims-1.9.1.jar	System Classpath
/opt/spark/jars/oro-2.0.8.jar	System Classpath
/opt/spark/jars/osgi-resource-locator-1.0.3.jar	System Classpath
/opt/spark/jars/paranamer-2.8.jar	System Classpath
/opt/spark/jars/parquet-column-1.13.1.jar	System Classpath
/opt/spark/jars/parquet-common-1.13.1.jar	System Classpath
/opt/spark/jars/parquet-encoding-1.13.1.jar	System Classpath
/opt/spark/jars/parquet-format-structures-1.13.1.jar	System Classpath
/opt/spark/jars/parquet-hadoop-1.13.1.jar	System Classpath
/opt/spark/jars/parquet-jackson-1.13.1.jar	System Classpath
/opt/spark/jars/pickle-1.3.jar	System Classpath
/opt/spark/jars/profiles-2.23.3.jar	System Classpath
/opt/spark/jars/protobuf-java-3.25.1.jar	System Classpath
/opt/spark/jars/protocol-core-2.23.3.jar	System Classpath
/opt/spark/jars/py4j-0.10.9.7.jar	System Classpath
/opt/spark/jars/reactive-streams-1.0.4.jar	System Classpath
/opt/spark/jars/regions-2.23.3.jar	System Classpath
/opt/spark/jars/rocksdbjni-8.3.2.jar	System Classpath
/opt/spark/jars/sbt-assembly_2.12_1.0-2.2.0.jar	System Classpath
/opt/spark/jars/sbt-native-packager_2.12_1.0-1.9.16.jar	System Classpath
/opt/spark/jars/scala-compiler-2.12.17.jar	System Classpath
/opt/spark/jars/scala-compiler-2.12.18.jar	System Classpath
/opt/spark/jars/scala-library-2.12.17.jar	System Classpath
/opt/spark/jars/scala-library-2.12.18.jar	System Classpath
/opt/spark/jars/scala-parser-combinators_2.12-1.1.2.jar	System Classpath
/opt/spark/jars/scala-parser-combinators_2.12-2.3.0.jar	System Classpath
/opt/spark/jars/scala-reflect-2.12.17.jar	System Classpath
/opt/spark/jars/scala-reflect-2.12.18.jar	System Classpath
/opt/spark/jars/scala-xml_2.12-2.1.0.jar	System Classpath
/opt/spark/jars/scalactic_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalamock_2.12-5.2.0.jar	System Classpath
/opt/spark/jars/scalatest-compatible-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-core_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-diagrams_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-featurespec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-flatspec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-freespec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-funspec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-funsuite_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-matchers-core_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-mustmatchers_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-propspec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-refspec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-shouldmatchers_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest-wordspec_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/scalatest_2.12-3.2.17.jar	System Classpath
/opt/spark/jars/sdk-core-2.23.3.jar	System Classpath
/opt/spark/jars/sentry-8.17.0.jar	System Classpath
/opt/spark/jars/shims-0.9.45.jar	System Classpath
/opt/spark/jars/slf4j-api-2.0.7.jar	System Classpath
/opt/spark/jars/snappy-java-1.1.10.5.jar	System Classpath
/opt/spark/jars/spark-catalyst_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-common-utils_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-core_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-kvstore_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-launcher_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-network-common_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-network-shuffle_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-sketch_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-sql-api_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-sql-kafka-0-10_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-sql_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-tags_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-token-provider-kafka-0-10_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/spark-unsafe_2.12-3.5.0.jar	System Classpath
/opt/spark/jars/sso-2.23.3.jar	System Classpath
/opt/spark/jars/ssooidc-2.23.3.jar	System Classpath
/opt/spark/jars/stream-2.9.6.jar	System Classpath
/opt/spark/jars/sts-2.23.3.jar	System Classpath
/opt/spark/jars/third-party-jackson-core-2.23.3.jar	System Classpath
/opt/spark/jars/threeten-extra-1.7.1.jar	System Classpath
/opt/spark/jars/tink-1.9.0.jar	System Classpath
/opt/spark/jars/tool-launcher-0.2.2.jar	System Classpath
/opt/spark/jars/univocity-parsers-2.9.1.jar	System Classpath
/opt/spark/jars/util-interface-1.9.4.jar	System Classpath
/opt/spark/jars/utils-2.23.3.jar	System Classpath
/opt/spark/jars/xbean-asm9-shaded-4.23.jar	System Classpath
/opt/spark/jars/xz-1.9.jar	System Classpath
/opt/spark/jars/zookeeper-3.6.3.jar	System Classpath
/opt/spark/jars/zookeeper-jute-3.6.3.jar	System Classpath
/opt/spark/jars/zstd-jni-1.5.5-4.jar	System Classpath
spark://07f6cd05c5e7:42015/jars/data_pipeline.jar	Added By User