Index: .gitignore =================================================================== diff -u --- .gitignore (revision 0) +++ .gitignore (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,4 @@ +.gradle +.idea +out +*.iml \ No newline at end of file Index: build.gradle =================================================================== diff -u --- build.gradle (revision 0) +++ build.gradle (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,60 @@ +buildscript { + ext { + springBootVersion = '2.5.5' + } + repositories { + mavenLocal() + maven { url 'https://workhorse.lemanscorp.com/nexus/repository/public/' } + } + dependencies { + classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") + } +} + +plugins { + id 'org.springframework.boot' version '2.5.5' + id 'io.spring.dependency-management' version '1.0.11.RELEASE' + id 'java' + id 'groovy' +} + +group = 'com.lemans' +version = '0.0.1-SNAPSHOT' +sourceCompatibility = '1.8' + +repositories { + mavenLocal() + maven { url 'https://workhorse.lemanscorp.com/nexus/repository/public/' } +} + +dependencies { + + implementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.85' + + implementation('org.codehaus.groovy:groovy') + implementation 'commons-dbcp:commons-dbcp:1.4' + implementation 'org.springframework.boot:spring-boot-starter-jdbc' + + implementation 'org.springframework.boot:spring-boot-starter-batch' + runtimeOnly 'com.microsoft.sqlserver:mssql-jdbc' + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.springframework.batch:spring-batch-test' + + implementation 'ch.qos.logback:logback-core' + implementation 'ch.qos.logback:logback-classic' + implementation 'net.logstash.logback:logstash-logback-encoder:4.11' + + compile group: 'com.h2database', name: 'h2' +} + +test { + useJUnitPlatform() +} + +sourceSets.main.java.srcDirs = [] +sourceSets.main.groovy.srcDirs += ["src/main/java"] + + +bootJar { + baseName = 'catalog-data-exporter' +} Index: gradle/wrapper/gradle-wrapper.jar =================================================================== diff -u Binary files differ Index: gradle/wrapper/gradle-wrapper.properties =================================================================== diff -u --- gradle/wrapper/gradle-wrapper.properties (revision 0) +++ gradle/wrapper/gradle-wrapper.properties (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.9-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists Index: gradlew =================================================================== diff -u --- gradlew (revision 0) +++ gradlew (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,237 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$(ls -ld "$app_path") + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$(cd "${APP_HOME:-./}" && pwd -P) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn() { + echo "$*" +} >&2 + +die() { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$(uname)" in #( +CYGWIN*) cygwin=true ;; #( +Darwin*) darwin=true ;; #( +MSYS* | MINGW*) msys=true ;; #( +NONSTOP*) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ]; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop"; then + case $MAX_FD in #( + max*) + MAX_FD=$(ulimit -H -n) || + warn "Could not query maximum file descriptor limit" + ;; + esac + case $MAX_FD in #( + '' | soft) : ;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + ;; + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys"; then + APP_HOME=$(cygpath --path --mixed "$APP_HOME") + CLASSPATH=$(cygpath --path --mixed "$CLASSPATH") + + JAVACMD=$(cygpath --unix "$JAVACMD") + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg; do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) + t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] + ;; #( + *) false ;; + esac + then + arg=$(cygpath --path --ignore --mixed "$arg") + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' +)" '"$@"' + +exec "$JAVACMD" "$@" Index: gradlew.bat =================================================================== diff -u --- gradlew.bat (revision 0) +++ gradlew.bat (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega Index: settings.gradle =================================================================== diff -u --- settings.gradle (revision 0) +++ settings.gradle (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1 @@ +rootProject.name = 'catalog-data-exporter' Index: src/main/java/com/lemans/CatalogDataExporterApplication.java =================================================================== diff -u --- src/main/java/com/lemans/CatalogDataExporterApplication.java (revision 0) +++ src/main/java/com/lemans/CatalogDataExporterApplication.java (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,12 @@ +package com.lemans; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CatalogDataExporterApplication { + + public static void main(String[] args) { + SpringApplication.run(CatalogDataExporterApplication.class, args); + } +} Index: src/main/java/com/lemans/Config.java =================================================================== diff -u --- src/main/java/com/lemans/Config.java (revision 0) +++ src/main/java/com/lemans/Config.java (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,66 @@ +package com.lemans; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.apache.commons.dbcp.BasicDataSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import javax.sql.DataSource; + +@Configuration +public class Config { + + @Value("${aws.accessKeyId}") + private String accessKey; + + @Value("${aws.secretKey}") + private String secretKey; + + @Value("${aws.region}") + private String region; + + @Primary + @Bean + @ConfigurationProperties(prefix = "spring.datasource") + BasicDataSource getDefaultDataSource() { + return new BasicDataSource(); + } + + @Primary + @Bean + NamedParameterJdbcTemplate getNamedParameterJdbcTemplate(DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } + + @Bean(name = "partsDataSource") + @ConfigurationProperties(prefix = "datasource.ps") + BasicDataSource getPartsDataSource() { + return new BasicDataSource(); + } + + @Bean(name = "partsJdbcTemplate") + NamedParameterJdbcTemplate getPartsJdbcTemplate(@Autowired @Qualifier("partsDataSource") DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } + + @Bean + public AWSCredentialsProvider credentialsProvider() { + return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)); + } + + @Bean + public AmazonS3 amazonS3Client() { + return AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(region)).withCredentials(credentialsProvider()).build(); + } +} Index: src/main/java/com/lemans/PartData.java =================================================================== diff -u --- src/main/java/com/lemans/PartData.java (revision 0) +++ src/main/java/com/lemans/PartData.java (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,151 @@ +package com.lemans; + +public class PartData { + + private String partNumber; + private String brand; + private String comcode; + private String whseCtry; + private String desc; + private String uom; + private String status; + private String hidden; + private String goliveDate; + private String inStockQuantity; + private String USvendor; + private String USvPartNumber; + private String UScarbCode; + private String EUvendor; + private String EUvPartNumber; + private String EUcertification; + + public PartData() { } + + public String getPartNumber() { + return partNumber; + } + + public void setPartNumber(String partNumber) { + this.partNumber = partNumber; + } + + public String getBrand() { + return brand; + } + + public void setBrand(String brand) { + this.brand = brand; + } + + public String getComcode() { + return comcode; + } + + public void setComcode(String comcode) { + this.comcode = comcode; + } + + public String getWhseCtry() { + return whseCtry; + } + + public void setWhseCtry(String whseCtry) { + this.whseCtry = whseCtry; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getUom() { + return uom; + } + + public void setUom(String uom) { + this.uom = uom; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getHidden() { + return hidden; + } + + public void setHidden(String hidden) { + this.hidden = hidden; + } + + public String getGoliveDate() { + return goliveDate; + } + + public void setGoliveDate(String goliveDate) { + this.goliveDate = goliveDate; + } + + public String getInStockQuantity() { + return inStockQuantity; + } + + public void setInStockQuantity(String inStockQuantity) { + this.inStockQuantity = inStockQuantity; + } + + public String getUSvendor() { + return USvendor; + } + + public void setUSvendor(String USvendor) { + this.USvendor = USvendor; + } + + public String getUSvPartNumber() { + return USvPartNumber; + } + + public void setUSvPartNumber(String USvPartNumber) { + this.USvPartNumber = USvPartNumber; + } + + public String getUScarbCode() { + return UScarbCode; + } + + public void setUScarbCode(String UScarbCode) { + this.UScarbCode = UScarbCode; + } + + public String getEUvendor() { + return EUvendor; + } + + public void setEUvendor(String EUvendor) { + this.EUvendor = EUvendor; + } + + public String getEUvPartNumber() { + return EUvPartNumber; + } + + public void setEUvPartNumber(String EUvPartNumber) { + this.EUvPartNumber = EUvPartNumber; + } + + public String getEUcertification() { + return EUcertification; + } + + public void setEUcertification(String EUcertification) { + this.EUcertification = EUcertification; + } +} Index: src/main/java/com/lemans/PartDataExporterJob.java =================================================================== diff -u --- src/main/java/com/lemans/PartDataExporterJob.java (revision 0) +++ src/main/java/com/lemans/PartDataExporterJob.java (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,198 @@ +package com.lemans; + +import com.amazonaws.services.s3.AmazonS3; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.listener.JobExecutionListenerSupport; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.JdbcCursorItemReader; +import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; +import org.springframework.batch.item.file.FlatFileHeaderCallback; +import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder; +import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor; +import org.springframework.batch.item.file.transform.DelimitedLineAggregator; +import org.springframework.batch.item.file.transform.FieldExtractor; +import org.springframework.batch.item.file.transform.LineAggregator; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.jdbc.core.PreparedStatementSetter; + +import javax.sql.DataSource; +import java.io.File; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.time.DateTimeException; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Date; + +@Configuration +@EnableBatchProcessing +@Import({ + Config.class, +}) +public class PartDataExporterJob { + + + private static final Logger log = LoggerFactory.getLogger(PartDataExporterJob.class); + + private static final String[] CSV_FIELDS = {"partNumber", "brand", "comcode", "whseCtry", "desc", "uom", "status", "hidden", "inStockQuantity", + "USvendor", "USvPartNumber", "UScarbCode", "EUvendor", "EUvPartNumber", "EUcertification"}; + private static final String CSV_DELIMITER = "~"; + + @Autowired + public JobBuilderFactory jobBuilderFactory; + + @Autowired + public StepBuilderFactory stepBuilderFactory; + + @Autowired + private AmazonS3 s3Client; + + @Value("${import.bucketName}") + private String bucketName; + + @Bean + protected Step processParts(ItemReader partReader, + ItemWriter partWriter) { + return stepBuilderFactory.get("processParts") + .chunk(10000) + .reader(partReader) + .writer(partWriter) + .build(); + } + + @Bean + protected Step moveFile() { + Tasklet tasklet = (stepContribution, chunkContext) -> { + String fileName = chunkContext.getStepContext().getJobExecutionContext().get("fileName").toString(); + s3Client.putObject(bucketName, fileName, new File(fileName)); + return RepeatStatus.FINISHED; + }; + return stepBuilderFactory.get("moveFilesToAWS") + .tasklet(tasklet) + .build(); + } + + @Bean + protected Step deleteLocalFile() { + Tasklet tasklet = (stepContribution, chunkContext) -> { + String fileName = chunkContext.getStepContext().getJobExecutionContext().get("fileName").toString(); + if (!new File(fileName).delete()) { + log.error("File not deleted " + fileName); + } + return RepeatStatus.FINISHED; + + }; + return stepBuilderFactory.get("deleteLocalFile") + .tasklet(tasklet) + .build(); + } + + @Bean + public JobExecutionListenerSupport listener() { + return new JobExecutionListenerSupport() { + + @Override + public void beforeJob(JobExecution jobExecution) { + log.error("!!! JOB Started!!!"); + String fileName = "Part_Data_" + new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss").format(new Date()) + ".csv"; + jobExecution.getExecutionContext().put("fileName", fileName); + jobExecution.getExecutionContext().put("queryDate", Date.from(LocalDateTime.now().minusYears(1).toInstant(ZoneOffset.UTC + ))); + } + + @Override + public void afterJob(JobExecution jobExecution) { + StepExecution execution = jobExecution.getStepExecutions().stream().findFirst().get(); + log.error("Processed following " + execution.getReadCount()); + log.error("!!! JOB Ended!!!"); + } + }; + } + + @Bean(destroyMethod = "") + public JdbcCursorItemReader partReader(@Qualifier("partsDataSource") DataSource partsDataSource) { + Date date = queryDate(null); + return new JdbcCursorItemReaderBuilder() + .name("partDataReader") + .sql(SqlQueries.PART_DATA) + .preparedStatementSetter(preparedStatement -> { + preparedStatement.setDate(1, new java.sql.Date(date.getTime())); + preparedStatement.setDate(2, new java.sql.Date(date.getTime())); + }) + .dataSource(partsDataSource) + .rowMapper(new BeanPropertyRowMapper<>(PartData.class)) + .build(); + } + + @Bean + public Job exportPartsJob(JobExecutionListenerSupport listener, Step processParts, Step moveFile, Step deleteLocalFile) { + return jobBuilderFactory.get("exportPartsJob") + .incrementer(new RunIdIncrementer()) + .listener(listener) + .start(processParts) + .next(moveFile) + .next(deleteLocalFile) + .build(); + } + + @Bean + @JobScope + public Resource resource(@Value("#{jobExecutionContext['fileName']}") String fileName) { + return new FileSystemResource(fileName); + } + + @Bean + @JobScope + public Date queryDate(@Value("#{jobExecutionContext['queryDate']}") Date date) { + return date; + } + + @Bean + public ItemWriter partWriter() { + FlatFileHeaderCallback headerCallback = writer -> writer.write(String.join(CSV_DELIMITER, CSV_FIELDS)); + LineAggregator lineAggregator = linesAggregator(); + return new FlatFileItemWriterBuilder() + .name("partsWriter") + .headerCallback(headerCallback) + .lineAggregator(lineAggregator) + .resource(resource(null)) + .build(); + } + + private LineAggregator linesAggregator() { + DelimitedLineAggregator lineAggregator = new DelimitedLineAggregator<>(); + lineAggregator.setDelimiter(CSV_DELIMITER); + FieldExtractor fieldExtractor = fieldsExtractor(); + lineAggregator.setFieldExtractor(fieldExtractor); + return lineAggregator; + } + + private FieldExtractor fieldsExtractor() { + BeanWrapperFieldExtractor extractor = new BeanWrapperFieldExtractor<>(); + extractor.setNames(CSV_FIELDS); + return extractor; + } +} Index: src/main/java/com/lemans/SqlQueries.java =================================================================== diff -u --- src/main/java/com/lemans/SqlQueries.java (revision 0) +++ src/main/java/com/lemans/SqlQueries.java (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,52 @@ +package com.lemans; + +public class SqlQueries { + + public static final String PART_DATA = "SELECT\n" + + " [partNumber] = p.punctuatedPartNumber,\n" + + " [brand] = p.brandCode,\n" + + " [comcode] = p.subComCode,\n" + + " [whseCtry] = p.warehouseCountry,\n" + + " [desc] = p.partDescr,\n" + + " [uom] = p.uom,\n" + + " [status] = p.partStatus,\n" + + " [hidden] = p.isHidden,\n" + + " [goliveDate] = dsp.goliveDate,\n" + + " [inStockQuantity] = IIF(dspa.natlAvail < 4, CAST(ISNULL(dspa.natlAvail, 0) AS varchar), '3+'),\n" + + " [USvendor] = p.vendorId,\n" + + " [USPartNumber] = p.vendorPartNumber,\n" + + " [USCarbCode] = p.carbCode,\n" + + " [EUvendor] = pk.vendorIdEu,\n" + + " [EUvPartNumber] = pk.vendorPartNumberEu,\n" + + " [EUcertification] = pk.certificationEu\n" + + "FROM\n" + + " (\n" + + " SELECT\n" + + " p.partNumber\n" + + " FROM\n" + + " dbo.Part p WITH(NOLOCK)\n" + + " WHERE\n" + + " p.locale = 'en'\n" + + " AND (p.lastModified >= ?)\n" + + " UNION\n" + + " SELECT\n" + + " pk.partNumber\n" + + " FROM\n" + + " dbo.vwPartKeyValue pk WITH(NOLOCK)\n" + + " WHERE\n" + + " (pk.lastModified >= ?)\n" + + ") cte\n" + + "INNER JOIN\n" + + "dbo.Part p WITH(NOLOCK)\n" + + " ON p.partNumber = cte.partNumber\n" + + " AND p.locale = 'en'\n" + + "LEFT OUTER JOIN\n" + + "dbo.vwPartKeyValue pk WITH(NOLOCK)\n" + + " ON pk.partNumber = p.partNumber\n" + + "LEFT OUTER JOIN\n" + + "PartsSource_DS.dbo.Part dsp WITH(NOLOCK)\n" + + " ON dsp.partNumber = p.partNumber\n" + + "LEFT OUTER JOIN\n" + + "PartsSource_DS.dbo.PartAvailability dspa WITH(NOLOCK)\n" + + " ON dspa.partNumber = p.partNumber"; +} Index: src/main/resources/application.properties =================================================================== diff -u --- src/main/resources/application.properties (revision 0) +++ src/main/resources/application.properties (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,22 @@ +aws.accessKeyId=AKIA5ENKY3BHTDKXIIHW +aws.secretKey=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 +aws.region=us-east-1 + +import.bucketName=lpu-dev + +spring.datasource.url=jdbc:h2:mem:testdb +spring.datasource.driverClassName=org.h2.Driver +spring.datasource.username=sa +spring.datasource.password= + + +datasource.ps.url=jdbc:sqlserver://dev-dbprod02vm;databaseName=Parts_US +datasource.ps.driverClassName=com.microsoft.sqlserver.jdbc.SQLServerDriver +datasource.ps.username=parts_us_user +datasource.ps.password=DevPassword1 + +spring.application.name=catalog-data-exporter + + +logging.level.com.microsoft.sqlserver.jdbc.Connection=TRACE +logging.level.com.microsoft.sqlserver.jdbc.Statement=TRACE \ No newline at end of file Index: src/test/java/com/lemans/CatalogDataExporterApplicationTests.java =================================================================== diff -u --- src/test/java/com/lemans/CatalogDataExporterApplicationTests.java (revision 0) +++ src/test/java/com/lemans/CatalogDataExporterApplicationTests.java (revision 61fc48c48ab2631fbce5712012f7a32ed5955a11) @@ -0,0 +1,13 @@ +package com.lemans; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class CatalogDataExporterApplicationTests { + + @Test + void contextLoads() { + } + +}