Spring Boot Oauth2 Refresh Token - IllegalStateException

So, I can get an access token all good with a standard CURL, but as soon as I try to get an access token the application throws an 'IllegalStateException - UserDetailsService Required'. As far as I knew we didnt need to authenticate user details again once we have refresh token? But anyway it should be there anyway considering it has to authenticate for the access token the first time around.

See below my Oauth2 config:

@Configuration
public class OAuth2Config {

@Configuration
@EnableResourceServer
protected static class ResourceServiceConfig extends ResourceServerConfigurerAdapter {

    @Autowired
    private TokenStore tokenStore;

    @Override
    public void configure(HttpSecurity http) throws Exception {
        http
            .authorizeRequests()
            .antMatchers("/oauth/token/").permitAll()
            .antMatchers(HttpMethod.GET, "/api/**").access("#oauth2.hasScope('read')")
            .antMatchers(HttpMethod.OPTIONS, "/api/**").access("#oauth2.hasScope('read')")
            .antMatchers(HttpMethod.POST, "/api/**").access("#oauth2.hasScope('write')")
            .antMatchers(HttpMethod.PUT, "/api/v1/**").access("#oauth2.hasScope('write')")
            .antMatchers(HttpMethod.PATCH, "/api/**").access("#oauth2.hasScope('write')")
            .antMatchers(HttpMethod.DELETE, "/api/**").access("#oauth2.hasScope('write')");
    }

    @Override
    public void configure(ResourceServerSecurityConfigurer resources) throws Exception {
        resources
            .resourceId(<RESOURCE ID>)
            .tokenStore(tokenStore);
    }
}

@Configuration
@EnableAuthorizationServer
protected static class AuthorizationServerConfig extends AuthorizationServerConfigurerAdapter {

    @Value("${oauth.clientid}")
    private String CLIENT_ID;

    @Value("${oauth.clientsecret}")
    private String CLIENT_SECRET;

    @Value("${oauth.tokenvalidity}")
    private String VALIDITY_SECONDS;

    @Autowired
    private DataSource dataSource;

    @Autowired
    private BCryptPasswordEncoder passwordEncoder;

    @Bean
    public TokenStore tokenStore() {
        return new JdbcTokenStore(dataSource);
    }

    @Autowired
    @Qualifier("authenticationManagerBean")
    private AuthenticationManager authenticationManager;

    @Override
    public void configure(AuthorizationServerEndpointsConfigurer endpoints) throws Exception {
        endpoints
            .tokenStore(tokenStore())
            .authenticationManager(authenticationManager);
    }

    @Override
    public void configure(ClientDetailsServiceConfigurer clients) throws Exception {
        clients
            .withClient(CLIENT_ID)
            .scopes("read", "write")
            .authorities(Authorities.USER)
            .authorizedGrantTypes("password", "refresh_token")
            .secret(CLIENT_SECRET)
            .accessTokenValiditySeconds(Integer.parseInt(VALIDITY_SECONDS));
    }

}
}

And Web Security Config :

@Configuration
@EnableWebSecurity
public class SecurityConfig extends WebSecurityConfigurerAdapter {

@Autowired
private CustomUserDetailsService userDetailsService;

@Bean
public BCryptPasswordEncoder passwordEncoder() {
    return new BCryptPasswordEncoder();
}


@Override
protected void configure(AuthenticationManagerBuilder auth) throws Exception {
    auth
        .userDetailsService(userDetailsService)
        .passwordEncoder(passwordEncoder());
}

@Override
@Bean
public AuthenticationManager authenticationManagerBean() throws Exception {
    return super.authenticationManagerBean();
}

@Override
public void configure(HttpSecurity http) throws Exception {
    http
        .csrf().disable()
        .authorizeRequests().anyRequest().permitAll();
}

}

So Yeah this one doesn't seem to make sense because 1. why is it even going for a user details service when we are attempting a refresh token grant? and 2. why can it not find the user details service anyway when it is clearly there and working for the password grant beforehand?

Thanks