[2017-08-24 10:41:27,047][WARN ][indices.cluster ] [Puff Adder] failed to clean index (removing index (no shards allocated)) org.elasticsearch.common.inject.ConfigurationException: Guice configuration errors: 1) A binding to org.elasticsearch.index.query.IndexQueryParserService already exists on a child injector. while locating org.elasticsearch.index.query.IndexQueryParserService 1 error at org.elasticsearch.common.inject.InjectorImpl.getProvider(InjectorImpl.java:851) at org.elasticsearch.common.inject.InjectorImpl.getProvider(InjectorImpl.java:790) at org.elasticsearch.common.inject.InjectorImpl.getInstance(InjectorImpl.java:862) at org.elasticsearch.indices.IndicesService.removeIndex(IndicesService.java:425) at org.elasticsearch.indices.IndicesService.removeIndex(IndicesService.java:381) at org.elasticsearch.indices.cluster.IndicesClusterStateService.removeIndex(IndicesClusterStateService.java:746) at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyCleanedIndices(IndicesClusterStateService.java:208) at org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:179) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:41:27,049][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:41:27,049][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:41:27,060][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:41:27,061][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:41:38,584][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:42:08,585][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:42:08,586][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:42:08,590][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:42:08,591][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:42:27,023][WARN ][rest.suppressed ] path: /megacorp/employee/1, params: {pretty=, index=megacorp, id=1, type=employee} UnavailableShardsException[[megacorp][3] primary shard is not active Timeout: [1m], request: [index {[megacorp][employee][1], source[ { "first_name" : "John", "last_name" : "Smith", "age" : 25, "about" : "I love to go rock climbing", "interests": [ "sports", "music" ] } ]}]] at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryBecauseUnavailable(TransportReplicationAction.java:614) at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:474) at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:576) at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:236) at org.elasticsearch.cluster.service.InternalClusterService$NotifyTimeout.run(InternalClusterService.java:816) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:42:38,587][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:43:08,588][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:43:08,588][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:43:08,591][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:43:08,592][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:43:38,589][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:44:08,590][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:44:08,590][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:44:08,593][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:44:08,594][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:44:38,592][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:45:08,593][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:45:08,593][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:45:08,596][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:45:08,597][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:45:38,594][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:46:08,595][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:46:08,595][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:46:08,598][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:46:08,599][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:46:38,596][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:47:08,597][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:47:08,597][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:47:08,600][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:47:08,601][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:47:38,598][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:48:08,599][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:48:08,600][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:48:08,602][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:48:08,604][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:48:38,601][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:49:08,602][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:49:08,602][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:49:08,605][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:49:08,606][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:49:38,603][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:50:08,605][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:50:08,605][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:50:08,607][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:50:08,608][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:50:38,606][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:51:08,607][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:51:08,607][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:51:08,610][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:51:08,611][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:51:38,608][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:52:08,609][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:52:08,609][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:52:08,613][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:52:08,614][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:52:38,610][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:53:08,611][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:53:08,612][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:53:08,615][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:53:08,616][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:53:38,613][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:54:08,614][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:54:08,614][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:54:08,619][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:54:08,620][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:54:38,615][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:55:08,616][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:55:08,616][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:55:08,619][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:55:08,620][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:55:38,617][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:56:08,618][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:56:08,618][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:56:08,621][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:56:08,622][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:56:38,620][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:57:08,621][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:57:08,621][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:57:08,624][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:57:08,625][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:57:38,622][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:58:08,623][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:58:08,623][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:58:08,626][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:58:08,627][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:58:38,625][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:59:08,626][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 10:59:08,626][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 10:59:08,628][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:59:08,629][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 10:59:38,627][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 11:00:08,628][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 11:00:08,628][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 11:00:08,631][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 11:00:08,632][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 11:00:38,629][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 11:01:08,630][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 11:01:08,630][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 11:01:08,633][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 11:01:08,634][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 11:01:38,631][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 11:02:08,632][WARN ][cluster.routing.allocation.decider] [Puff Adder] high disk watermark [90%] exceeded on [5fTMhL_6RAWHFZvsh83QKg][Puff Adder][/usr/share/elasticsearch/data/elasticsearch/nodes/0] free: 0b[0%], shards will be relocated away from this node [2017-08-24 11:02:08,632][INFO ][cluster.routing.allocation.decider] [Puff Adder] rerouting shards: [high disk watermark exceeded on one or more nodes] [2017-08-24 11:02:08,635][WARN ][gateway ] [Puff Adder] [test-index]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2017-08-24 11:02:08,636][WARN ][gateway ] [Puff Adder] [megacorp]: failed to write index state java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices: No space left on device at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:384) at java.nio.file.Files.createDirectory(Files.java:674) at java.nio.file.Files.createAndCheckIsDirectory(Files.java:781) at java.nio.file.Files.createDirectories(Files.java:767) at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:107) at org.elasticsearch.gateway.MetaStateService.writeIndex(MetaStateService.java:135) at org.elasticsearch.gateway.GatewayMetaState.clusterChanged(GatewayMetaState.java:160) at org.elasticsearch.gateway.Gateway.clusterChanged(Gateway.java:185) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)