java - How to avoid Spring batch persistence of metadata in DB

Question

I have a Spring batch job that reads from a DB and writes into a CSV. The batch job is trying to use the DB from which I am reading to save the status of the batch processing. I don't want this to happen. I am using spring 4.

After extensive googling, I tried 2 approaches.But both of them failed.

Approach 1:

@Bean
    public JobLauncher jobLauncher() {
        SimpleJobLauncher simpleJobLauncher = new SimpleJobLauncher();
        simpleJobLauncher.setJobRepository(jobRepository());
        simpleJobLauncher.setTaskExecutor(taskExecutor());
        return simpleJobLauncher;
    }

    @Bean
    public JobRepository jobRepository() {
        SimpleJobRepository simpleJobRepository = new SimpleJobRepository(new MapJobInstanceDao(),
                new MapJobExecutionDao(), new MapStepExecutionDao(), new MapExecutionContextDao());
        return simpleJobRepository;
    }

    @Bean
    public SimpleJob simpleJob() {
        SimpleJob simpleJob = new SimpleJob();
        simpleJob.setName("Feedback surevey job");
        List<Step> steps = new ArrayList<>(1);
        steps.add(step1());
        simpleJob.setSteps(steps);
        simpleJob.setJobRepository(jobRepository());
        return simpleJob;
    }

    @Bean
    public TaskExecutor taskExecutor(){
        SimpleAsyncTaskExecutor asyncTaskExecutor=new SimpleAsyncTaskExecutor("spring_batch");
        asyncTaskExecutor.setConcurrencyLimit(this.threadCount);
        return asyncTaskExecutor;
    }

    @Bean
    public Step step1() {
        return stepBuilderFactory.get("step1").<Feedback, Feedback> chunk(chuckSize)
                .reader(reader())
                .writer(writer())
                .taskExecutor(taskExecutor())
                .throttleLimit(this.threadCount).build();
    }

Approach 2:

    @Bean
    public Job importUserJob(JobCompletionNotificationListener listener) {
          return jobBuilderFactory.get("importUserJob")
                           .incrementer(new RunIdIncrementer()).listener(listener)
                            .flow(step1()).end().build();
    }

   @Bean
   public ResourcelessTransactionManager transactionManager() {
       return new ResourcelessTransactionManager();
   }

   @Bean
   public JobRepository jobRepository(ResourcelessTransactionManager transactionManager) throws Exception {
       MapJobRepositoryFactoryBean mapJobRepositoryFactoryBean = new MapJobRepositoryFactoryBean(transactionManager);
       mapJobRepositoryFactoryBean.setTransactionManager(transactionManager);
       return mapJobRepositoryFactoryBean.getObject();
   }

    @Bean
    public Step step1() {
    return stepBuilderFactory.get("step1").<Feedback, Feedback> chunk(chuckSize)
            .reader(reader())
            .writer(writer())
            .taskExecutor(taskExecutor())
            .throttleLimit(this.threadCount).build();
    }

Relevant part in the application.properties:

spring.datasource.url=######
spring.datasource.username=************
spring.datasource.password=$$$$$$$$$$

I took reference from the following posts

• https://dzone.com/articles/spring-batch-hello-world-1 Spring-Batch without persisting metadata to database?
• Spring Batch - How to prevent batch from storing transactions in DB
• Spring Boot Batch ResourcelessTransactionManager DataSourceProperties$DataSourceBeanCreationException

The exception I get is as follows. Though the exception is not a show stopper, I would like to get rid of it.

java.sql.SQLSyntaxErrorException: ORA-01031: insufficient privileges

    at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:450) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:399) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4C8Oall.processError(T4C8Oall.java:1017) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:655) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:249) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:566) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:202) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:45) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CStatement.executeForRows(T4CStatement.java:933) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1075) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.OracleStatement.executeInternal(OracleStatement.java:1718) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.OracleStatement.execute(OracleStatement.java:1678) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.OracleStatementWrapper.execute(OracleStatementWrapper.java:332) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_121]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_121]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_121]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_121]
    at org.apache.tomcat.jdbc.pool.StatementFacade$StatementProxy.invoke(StatementFacade.java:114) ~[tomcat-jdbc-8.5.27.jar:na]
    at com.sun.proxy.$Proxy50.execute(Unknown Source) ~[na:na]
    at org.springframework.jdbc.datasource.init.ScriptUtils.executeSqlScript(ScriptUtils.java:470) ~[spring-jdbc-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.jdbc.datasource.init.ResourceDatabasePopulator.populate(ResourceDatabasePopulator.java:238) [spring-jdbc-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.jdbc.datasource.init.DatabasePopulatorUtils.execute(DatabasePopulatorUtils.java:48) [spring-jdbc-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.boot.autoconfigure.AbstractDatabaseInitializer.initialize(AbstractDatabaseInitializer.java:66) [spring-boot-autoconfigure-1.5.10.RELEASE.jar:1.5.10.RELEASE]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_121]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_121]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_121]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_121]
    at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:366) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:311) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:134) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:409) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1620) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:555) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:306) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:302) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:761) [spring-beans-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:867) [spring-context-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:543) [spring-context-4.3.14.RELEASE.jar:4.3.14.RELEASE]
    at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:693) [spring-boot-1.5.10.RELEASE.jar:1.5.10.RELEASE]
    at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:360) [spring-boot-1.5.10.RELEASE.jar:1.5.10.RELEASE]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:303) [spring-boot-1.5.10.RELEASE.jar:1.5.10.RELEASE]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:1118) [spring-boot-1.5.10.RELEASE.jar:1.5.10.RELEASE]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:1107) [spring-boot-1.5.10.RELEASE.jar:1.5.10.RELEASE]
    at hello.Application.main(Application.java:10) [classes/:na]

Error Log:

2018-02-23 16:29:02.211 DEBUG 9688 --- [           main] o.s.jdbc.datasource.DataSourceUtils      : Fetching JDBC Connection from DataSource
2018-02-23 16:29:02.211  INFO 9688 --- [           main] o.s.jdbc.datasource.init.ScriptUtils     : Executing SQL script from class path resource [org/springframework/batch/core/schema-oracle10g.sql]
2018-02-23 16:29:02.772 DEBUG 9688 --- [           main] o.s.jdbc.datasource.init.ScriptUtils     : Failed to execute SQL script statement #1 of class path resource [org/springframework/batch/core/schema-oracle10g.sql]: CREATE TABLE BATCH_JOB_INSTANCE ( JOB_INSTANCE_ID NUMBER(19,0) NOT NULL PRIMARY KEY , VERSION NUMBER(19,0) , JOB_NAME VARCHAR2(100) NOT NULL, JOB_KEY VARCHAR2(32) NOT NULL, constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY) ) 

java.sql.SQLSyntaxErrorException: ORA-01031: insufficient privileges

    at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:450) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:399) ~[ojdbc7-12.1.0.1.jar:12.1.0.1.0]
    at oracle.jdbc.driver.T4C8Oall.processError(T4C8Oall.java:

Answer 1

As far as I know, disabling metadata persistence is not possible. A possible workaround for not having to setup a 'proper' database is to use an in-memory database for the metadata:

pom.xml

    <dependency>
        <groupId>com.h2database</groupId>
        <artifactId>h2</artifactId>
    </dependency>

However, there is the problem of using the default Spring Datasource for the Spring Batch Jobs Metadata repository. Here there is a complete workaround in order to define a secondary datasource for it:

application.properties

###################################
### JOBS DATASOURCE PROPERTIES. ###
###################################
## URL used to connect to the jobs database.
spring.secondDatasource.url=jdbc:h2:mem:jobsdb;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
## Driver class used to connect to the jobs database (it will depend on datasource).
spring.secondDatasource.driver-class-Name=org.h2.Driver
## User name 
spring.secondDatasource.username=xxx
## Password 
spring.secondDatasource.password=xxx
## Datasource configuration for jobs database.
spring.jpa.hibernate.ddl-auto=create-drop
spring.secondDatasource.initialize=true
spring.secondDatasource.test-on-borrow=true
spring.secondDatasource.validation-query=select 1

Spring Configuration (I) Datasources

/**
 * Config class holding several datasources, one business related, other for the spring batch jobs
 */
@Configuration
public class DataSourceConfiguration
{
    @Bean
    @Qualifier("businessDataSource")
    @ConfigurationProperties(prefix = "spring.datasource")
    public DataSource primaryDataSource()
    {
        return DataSourceBuilder.create().build();
    }

    @Bean
    @Primary
    @Qualifier("jobsDataSource")
    @ConfigurationProperties(prefix = "spring.secondDatasource")
    public DataSource secondaryDataSource()
    {
        return DataSourceBuilder.create().build();
    }
}

Spring Configuration (II) - Spring batch

import org.springframework.batch.core.configuration.annotation.DefaultBatchConfigurer;

// Another @Configuration class...

@Autowired
@Qualifier("jobsDataSource")
private DataSource dataSource;

@Bean
public BatchConfigurer configurer()
{
    // This is required to avoid problems when jobs datasource is into some secondary datasource.
    return new DefaultBatchConfigurer(dataSource);
}

With all this, Spring Batch will use the in-memory datasource, while you are free to use the default datasource for your own purposes