说明
线上出现异常,但是查看节点状态正常,因为使用了容器,挂掉了会重启。因为jvm配置了-XX:HeapDumpOnOutOfMemoryError 参数,出现OMM就会将当时线程和jvm内存情况转存起来可参考:jmv参数配置
排查
1.将运维发过来的hprof导入到 vm 可参考 直通车
3.我们去当时的线程日志查看
4.搜索13219这个线程id找到线程查看堆栈信息
5.分析堆栈找到业务代码 分析问题原因
具体看标红处 发现是导入功能问题 导致oom
"http-nio-8080-exec-11" daemon prio=5 tid=13219 RUNNABLE at java.lang.OutOfMemoryError.<init>(OutOfMemoryError.java:48) at org.apache.xmlbeans.impl.store.CharUtil.allocate(CharUtil.java:397) local variable: org.apache.xmlbeans.impl.store.CharUtil#2 at org.apache.xmlbeans.impl.store.CharUtil.saveChars(CharUtil.java:506) local variable: org.apache.xmlbeans.impl.store.CharUtil#2 local variable: java.lang.String#622892 local variable: char[]#2750421 at org.apache.xmlbeans.impl.store.Cur$CurLoadContext.text(Cur.java:2927) local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 at org.apache.xmlbeans.impl.store.Cur$CurLoadContext.attr(Cur.java:3048) local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 local variable: java.lang.String#622892 local variable: org.apache.xmlbeans.impl.store.Xobj$AttrXobj#2119297 at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1440) local variable: org.apache.xmlbeans.impl.store.Locale#3 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1062106 local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 local variable: com.sun.org.apache.xerces.internal.dom.AttributeMap#1061610 at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403) local variable: org.apache.xmlbeans.impl.store.Locale#3 local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1062106 at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445) local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403) local variable: org.apache.xmlbeans.impl.store.Locale#3 local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#243777 at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445) local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403) local variable: org.apache.xmlbeans.impl.store.Locale#3 local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1 at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445) local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 at org.apache.xmlbeans.impl.store.Locale.parseToXmlObject(Locale.java:1385) local variable: org.apache.xmlbeans.impl.store.Locale#3 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2 local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3 local variable: org.apache.xmlbeans.XmlOptions#3 local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1 at org.apache.xmlbeans.impl.store.Locale.parseToXmlObject(Locale.java:1370) local variable: org.apache.xmlbeans.impl.schema.SchemaTypeLoaderImpl#1 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2 local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3 local variable: org.apache.xmlbeans.XmlOptions#3 local variable: org.apache.xmlbeans.impl.store.Locale#3 local variable: org.apache.xmlbeans.impl.store.Locale#3 at org.apache.xmlbeans.impl.schema.SchemaTypeLoaderBase.parse(SchemaTypeLoaderBase.java:370) local variable: org.apache.xmlbeans.impl.schema.SchemaTypeLoaderImpl#1 local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2 local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3 local variable: org.apache.xmlbeans.XmlOptions#3 at org.apache.poi.POIXMLTypeLoader.parse(POIXMLTypeLoader.java:164) local variable: java.io.ByteArrayInputStream#1 local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3 local variable: org.apache.xmlbeans.XmlOptions#3 local variable: com.sun.org.apache.xerces.internal.dom.DeferredDocumentImpl#50 at org.openxmlformats.schemas.spreadsheetml.x2006.main.WorksheetDocument$Factory.parse(<unresolved string 0x0>) local variable: java.io.ByteArrayInputStream#1 local variable: org.apache.xmlbeans.XmlOptions#3 at org.apache.poi.xssf.usermodel.XSSFSheet.read(XSSFSheet.java:226) local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1 local variable: java.io.ByteArrayInputStream#1 at org.apache.poi.xssf.usermodel.XSSFSheet.onDocumentRead(XSSFSheet.java:218) local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1 at org.apache.poi.xssf.usermodel.XSSFWorkbook.parseSheet(XSSFWorkbook.java:443) local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1 local variable: java.util.HashMap#17405 local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetImpl#1 local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1 at org.apache.poi.xssf.usermodel.XSSFWorkbook.onDocumentRead(XSSFWorkbook.java:408) local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1 local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.WorkbookDocumentImpl#1 local variable: org.apache.poi.xssf.model.ThemesTable#1 local variable: java.util.HashMap#17405 local variable: java.util.HashMap#17406 local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSheet[]#1 local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetImpl#1 at org.apache.poi.POIXMLDocument.load(POIXMLDocument.java:169) local variable: java.util.HashMap#17407 at org.apache.poi.xssf.usermodel.XSSFWorkbook.<init>(XSSFWorkbook.java:300) local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1 at com.biaoguoworks.commons.utils.ExcelUtils.readExcel(ExcelUtils.java:204) local variable: class com.biaoguoworks.scm.logistics.dto.sorting.task.SkuSortingImportDTO local variable: java.lang.String#254630 local variable: java.util.ArrayList#63983 local variable: java.io.FileInputStream#8 at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController.sortingImport(SortingTagController.java:168) local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController#1 local variable: com.biaoguoworks.scm.logistics.vo.sorting.SkuSortingImportVO#1 at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$FastClassBySpringCGLIB$$9eba9d0d.invoke(<generated>) at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) local variable: org.springframework.cglib.proxy.MethodProxy#4788 at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749) local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1 at org.springframework.validation.beanvalidation.MethodValidationInterceptor.invoke(MethodValidationInterceptor.java:119) local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1 local variable: java.lang.Class[]#5445 local variable: org.hibernate.validator.internal.engine.ValidatorImpl#1 local variable: java.lang.reflect.Method#91741 at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749) local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1 at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691) local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6#1 local variable: java.lang.reflect.Method#91741 local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController#1 local variable: org.springframework.aop.target.SingletonTargetSource#57 at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6.sortingImport(<generated>) local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6#1 local variable: com.biaoguoworks.scm.logistics.vo.sorting.SkuSortingImportVO#1 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190) local variable: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod#1 local variable: java.lang.Object[]#89529 at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105) local variable: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod#1 local variable: org.springframework.web.context.request.ServletWebRequest#1 local variable: org.springframework.web.method.support.ModelAndViewContainer#1 at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878) local variable: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter#1 local variable: org.springframework.web.context.request.ServletWebRequest#1 local variable: org.springframework.web.method.annotation.ModelFactory#1 local variable: org.springframework.web.method.support.ModelAndViewContainer#1 local variable: org.springframework.web.context.request.async.WebAsyncManager#1 at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792) local variable: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter#1 local variable: org.apache.catalina.connector.ResponseFacade#5 local variable: org.springframework.web.method.HandlerMethod#378 at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040) local variable: org.springframework.web.servlet.DispatcherServlet#1 local variable: org.apache.catalina.connector.ResponseFacade#5 local variable: org.springframework.web.multipart.support.StandardMultipartHttpServletRequest#1 local variable: org.springframework.web.servlet.HandlerExecutionChain#1 local variable: org.springframework.web.context.request.async.WebAsyncManager#1 at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943) local variable: org.springframework.web.servlet.DispatcherServlet#1 local variable: org.apache.catalina.connector.RequestFacade#5 at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) local variable: org.springframework.web.servlet.DispatcherServlet#1 local variable: org.apache.catalina.connector.RequestFacade#5 local variable: org.apache.catalina.connector.ResponseFacade#5 local variable: org.springframework.context.i18n.SimpleLocaleContext#1 local variable: org.springframework.web.context.request.ServletRequestAttributes#2 local variable: org.springframework.web.context.request.ServletRequestAttributes#1 local variable: org.springframework.web.context.request.async.WebAsyncManager#1 at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909) at javax.servlet.http.HttpServlet.service(HttpServlet.java:652) at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) at javax.servlet.http.HttpServlet.service(HttpServlet.java:733) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at com.github.xiaoymin.knife4j.spring.filter.SecurityBasicAuthFilter.doFilter(SecurityBasicAuthFilter.java:90) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at com.github.xiaoymin.knife4j.spring.filter.ProductionSecurityFilter.doFilter(ProductionSecurityFilter.java:53) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) local variable: org.springframework.boot.web.servlet.filter.OrderedRequestContextFilter#1 local variable: org.apache.catalina.connector.RequestFacade#5 local variable: org.springframework.web.context.request.ServletRequestAttributes#2 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) local variable: org.apache.catalina.connector.RequestFacade#5 local variable: java.lang.String#254631 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) local variable: org.apache.catalina.connector.RequestFacade#5 local variable: java.lang.String#254632 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) local variable: org.apache.catalina.connector.RequestFacade#5 local variable: java.lang.String#254633 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202) local variable: org.apache.catalina.core.StandardWrapperValve#2 local variable: org.apache.catalina.connector.Request#5 local variable: org.apache.catalina.connector.Response#5 local variable: org.apache.catalina.core.StandardWrapper#2 local variable: org.springframework.web.servlet.DispatcherServlet#1 local variable: org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedContext#1 local variable: org.apache.catalina.core.ApplicationFilterChain#5 local variable: org.apache.catalina.core.StandardWrapper#2 at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139) local variable: org.apache.catalina.core.StandardHostValve#1 local variable: org.apache.catalina.connector.Request#5 local variable: org.apache.catalina.connector.Response#5 local variable: org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedContext#1 at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) local variable: org.apache.catalina.valves.ErrorReportValve#1 local variable: org.apache.catalina.connector.Request#5 local variable: org.apache.catalina.connector.Response#5 at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74) at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:747) local variable: org.apache.catalina.valves.RemoteIpValve#1 local variable: org.apache.catalina.connector.Request#5 local variable: java.lang.String#254634 local variable: java.lang.String#254634 local variable: java.lang.String#18896 local variable: java.lang.String#254635 local variable: java.lang.String#254637 local variable: java.lang.String#254636 at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343) local variable: org.apache.catalina.connector.CoyoteAdapter#1 local variable: org.apache.coyote.Request#5 local variable: org.apache.coyote.Response#5 local variable: org.apache.catalina.connector.Request#5 local variable: org.apache.catalina.connector.Response#5 at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:373) local variable: org.apache.coyote.http11.Http11Processor#5 local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7 local variable: org.apache.coyote.RequestInfo#5 at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65) local variable: org.apache.coyote.http11.Http11Processor#5 local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7 local variable: org.apache.tomcat.util.net.SocketEvent#6 at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868) local variable: org.apache.coyote.AbstractProtocol$ConnectionHandler#1 local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7 local variable: org.apache.tomcat.util.net.SocketEvent#6 local variable: org.apache.coyote.http11.Http11Processor#5 at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1589) local variable: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor#4 local variable: org.apache.tomcat.util.net.NioChannel#17 local variable: org.apache.tomcat.util.net.NioEndpoint$Poller#1 at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) local variable: org.apache.tomcat.util.threads.ThreadPoolExecutor#1 local variable: java.util.concurrent.ThreadPoolExecutor$Worker#63 local variable: org.apache.tomcat.util.threads.TaskThread#14 local variable: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor#4 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) local variable: org.apache.tomcat.util.threads.TaskThread$WrappingRunnable#14 at java.lang.Thread.run(Thread.java:748)