No adapter for handler TokenEndpoint spring security

I am trying to configure my Jersey REST API to work with spring security oauth2. However when I try to make a call to:

http://localhost:8080/oauth/token?grant_type=password&username=be@gmail.com&password=123456

I get the error below:

No adapter for handler [public org.springframework.http.ResponseEntity org.springframework.security.oauth2.provider.endpoint.TokenEndpoint.getAccessToken(java.security.Principal,java.util.Map,org.springframework.http.HttpMethod)

I do see spring-mvc jar in my libs, as if I remove it I get an annotation error.

What is wrong?

I don't want to use spring-mvc - my project is a standalone Jersey REST API that I would like to secure with tokens and oauth2.

Oauth configuration:

@Configuration
@EnableAuthorizationServer
public class OAuthConfiguration {

    @Configuration
    @EnableAuthorizationServer
    protected static class AuthorizationServerConfiguration extends AuthorizationServerConfigurerAdapter {

        @Autowired
        private DataSource dataSource;

        @Bean
        public TokenStore tokenStore() {
            return new JdbcTokenStore(dataSource);
        }

        @Override
        public void configure(AuthorizationServerEndpointsConfigurer endpoints) throws Exception {
            endpoints.tokenStore(tokenStore());
        }

        @Override
        public void configure(ClientDetailsServiceConfigurer clients) throws Exception {
            clients.jdbc(dataSource);
        }

    }

    @Configuration
    @EnableResourceServer
    protected static class ResourceServerConfiguration extends ResourceServerConfigurerAdapter {

        @Autowired
        private DataSource dataSource;

        @Bean
        public TokenStore tokenStore() {
            return new JdbcTokenStore(dataSource);
        }

        @Override
        public void configure(ResourceServerSecurityConfigurer resources) throws Exception {
            resources.resourceId("api1").tokenStore(tokenStore());
        }

        @Override
        public void configure(HttpSecurity http) throws Exception {
            // @formatter:off
            http
            .authorizeRequests()
            .antMatchers(HttpMethod.GET, "/**").access("#oauth2.hasScope('read')")
            .antMatchers(HttpMethod.OPTIONS, "/**").access("#oauth2.hasScope('read')")
            .antMatchers(HttpMethod.POST, "/**").access("#oauth2.hasScope('write')")
            .antMatchers(HttpMethod.PUT, "/**").access("#oauth2.hasScope('write')")
            .antMatchers(HttpMethod.DELETE, "/**").access("#oauth2.hasScope('write')");
            // @formatter:on
        }

    }
}

web configuration:

@Configuration
@EnableWebSecurity
public class SecurityConfiguration extends WebSecurityConfigurerAdapter {

    @Autowired
    private UserManager userManager;

    @Override
    protected void configure(AuthenticationManagerBuilder auth) throws Exception {
        auth.userDetailsService(userManager);
    }

    @Override
    protected void configure(HttpSecurity http) throws Exception {
        http.authorizeRequests().antMatchers("/**").authenticated().and().httpBasic().realmName("OAuth Server");
    }
}

and web.xml:

<?xml version="1.0" encoding="UTF-8"?>
<web-app version="3.0" xmlns="http://ift.tt/nSRXKP"
    xmlns:xsi="http://ift.tt/ra1lAU"
    xsi:schemaLocation="http://ift.tt/nSRXKP http://ift.tt/1eWqHMP">


    <context-param>
        <param-name>contextClass</param-name>
        <param-value>        org.springframework.web.context.support.AnnotationConfigWebApplicationContext
        </param-value>
    </context-param>
    <context-param>
        <param-name>contextConfigLocation</param-name>
        <param-value>com.estartup.config.PersistenceConfig</param-value>
    </context-param>

    <listener>
        <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
    </listener>

    <servlet>
        <servlet-name>jersey-servlet</servlet-name>
        <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
        <init-param>
            <param-name>javax.ws.rs.Application</param-name>
            <param-value>com.estartup.config.AlgopixApplication</param-value>
        </init-param>

        <load-on-startup>1</load-on-startup>
    </servlet>
    <servlet-mapping>
        <servlet-name>jersey-servlet</servlet-name>
        <url-pattern>/api/*</url-pattern>
    </servlet-mapping>

    <servlet>
        <servlet-name>spring</servlet-name>
        <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
        <load-on-startup>2</load-on-startup>
    </servlet>

    <servlet-mapping>
        <servlet-name>spring</servlet-name>
        <url-pattern>/oauth/token</url-pattern>
    </servlet-mapping>
</web-app>

What is wrong? I am spending so much time to configure jersey oauth2 and spring via annotations :( that I am afraid that it will take me less to write the code by myself :( but so many ppl are using spring security