- java.lang.Object
-
- org.apache.lucene.index.DocumentsWriterFlushControl
-
- All Implemented Interfaces:
java.io.Closeable
,java.lang.AutoCloseable
,Accountable
final class DocumentsWriterFlushControl extends java.lang.Object implements Accountable, java.io.Closeable
This class controlsDocumentsWriterPerThread
flushing during indexing. It tracks the memory consumption perDocumentsWriterPerThread
and uses a configuredFlushPolicy
to decide if aDocumentsWriterPerThread
must flush.In addition to the
FlushPolicy
the flush control might set certainDocumentsWriterPerThread
as flush pending iff aDocumentsWriterPerThread
exceeds theIndexWriterConfig.getRAMPerThreadHardLimitMB()
to prevent address space exhaustion.
-
-
Field Summary
Fields Modifier and Type Field Description private long
activeBytes
private java.util.Queue<DocumentsWriterPerThread>
blockedFlushes
private boolean
closed
private LiveIndexWriterConfig
config
private DocumentsWriter
documentsWriter
private boolean
flushByRAMWasDisabled
private long
flushBytes
private java.util.concurrent.atomic.AtomicBoolean
flushDeletes
private java.util.List<DocumentsWriterPerThread>
flushingWriters
private FlushPolicy
flushPolicy
private java.util.Queue<DocumentsWriterPerThread>
flushQueue
private boolean
fullFlush
private boolean
fullFlushMarkDone
private long
hardMaxBytesPerDWPT
private InfoStream
infoStream
private double
maxConfiguredRamBuffer
private int
numDocsSinceStalled
private int
numPending
private long
peakActiveBytes
private long
peakDelta
private long
peakFlushBytes
private long
peakNetBytes
private DocumentsWriterPerThreadPool
perThreadPool
(package private) DocumentsWriterStallControl
stallControl
private long
stallStartNS
-
Fields inherited from interface org.apache.lucene.util.Accountable
NULL_ACCOUNTABLE
-
-
Constructor Summary
Constructors Constructor Description DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config)
-
Method Summary
All Methods Instance Methods Concrete Methods Modifier and Type Method Description (package private) void
abortFullFlushes()
(package private) void
abortPendingFlushes()
long
activeBytes()
private void
addFlushingDWPT(DocumentsWriterPerThread perThread)
java.util.Iterator<DocumentsWriterPerThread>
allActiveWriters()
Returns an iterator that provides access to all currently activeDocumentsWriterPerThread
s(package private) boolean
anyStalledThreads()
Returnstrue
iff stalledprivate boolean
assertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)
(package private) boolean
assertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)
private boolean
assertMemory()
private boolean
assertNumDocsSinceStalled(boolean stalled)
private DocumentsWriterPerThread
checkout(DocumentsWriterPerThread perThread, boolean markPending)
private void
checkoutAndBlock(DocumentsWriterPerThread perThread)
To be called only by the owner of this object's monitor lockprivate DocumentsWriterPerThread
checkOutForFlush(DocumentsWriterPerThread perThread)
(package private) DocumentsWriterPerThread
checkoutLargestNonPendingWriter()
Returns the largest non-pending flushable DWPT ornull
if there is none.void
close()
(package private) DocumentsWriterPerThread
doAfterDocument(DocumentsWriterPerThread perThread)
(package private) void
doAfterFlush(DocumentsWriterPerThread dwpt)
(package private) void
doOnAbort(DocumentsWriterPerThread perThread)
(package private) void
doOnDelete()
(package private) DocumentsWriterPerThread
findLargestNonPendingWriter()
(package private) void
finishFullFlush()
boolean
getAndResetApplyAllDeletes()
long
getDeleteBytesUsed()
Returns heap bytes currently consumed by buffered deletes/updates that would be freed if we pushed all deletes.(package private) long
getFlushingBytes()
InfoStream
getInfoStream()
Returns theIndexWriter
InfoStream
(package private) long
getPeakActiveBytes()
(package private) long
getPeakNetBytes()
(package private) boolean
isFullFlush()
Returnstrue
if a full flush is currently running(package private) long
markForFullFlush()
(package private) long
netBytes()
(package private) DocumentsWriterPerThread
nextPendingFlush()
(package private) int
numBlockedFlushes()
Returns the number of flushes that are checked out but not yet available for flushing.(package private) int
numFlushingDWPT()
(package private) int
numQueuedFlushes()
Returns the number of flushes that are already checked out but not yet actively flushing(package private) DocumentsWriterPerThread
obtainAndLock()
private void
pruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
Prunes the blockedQueue by removing all DWPTs that are associated with the given flush queue.private long
ramBufferGranularity()
Return the smallest number of bytes that we would like to make sure to not miss from the global RAM accounting.long
ramBytesUsed()
Return the memory usage of this object in bytes.void
setApplyAllDeletes()
void
setFlushPending(DocumentsWriterPerThread perThread)
Sets flush pending state on the givenDocumentsWriterPerThread
.private long
stallLimitBytes()
java.lang.String
toString()
private boolean
updatePeaks(long delta)
private boolean
updateStallState()
void
waitForFlush()
(package private) void
waitIfStalled()
This method will block if too many DWPT are currently flushing and no checked out DWPT are available-
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
-
Methods inherited from interface org.apache.lucene.util.Accountable
getChildResources
-
-
-
-
Field Detail
-
hardMaxBytesPerDWPT
private final long hardMaxBytesPerDWPT
-
activeBytes
private long activeBytes
-
flushBytes
private volatile long flushBytes
-
numPending
private volatile int numPending
-
numDocsSinceStalled
private int numDocsSinceStalled
-
flushDeletes
private final java.util.concurrent.atomic.AtomicBoolean flushDeletes
-
fullFlush
private boolean fullFlush
-
fullFlushMarkDone
private boolean fullFlushMarkDone
-
flushQueue
private final java.util.Queue<DocumentsWriterPerThread> flushQueue
-
blockedFlushes
private final java.util.Queue<DocumentsWriterPerThread> blockedFlushes
-
flushingWriters
private final java.util.List<DocumentsWriterPerThread> flushingWriters
-
maxConfiguredRamBuffer
private double maxConfiguredRamBuffer
-
peakActiveBytes
private long peakActiveBytes
-
peakFlushBytes
private long peakFlushBytes
-
peakNetBytes
private long peakNetBytes
-
peakDelta
private long peakDelta
-
flushByRAMWasDisabled
private boolean flushByRAMWasDisabled
-
stallControl
final DocumentsWriterStallControl stallControl
-
perThreadPool
private final DocumentsWriterPerThreadPool perThreadPool
-
flushPolicy
private final FlushPolicy flushPolicy
-
closed
private boolean closed
-
documentsWriter
private final DocumentsWriter documentsWriter
-
config
private final LiveIndexWriterConfig config
-
infoStream
private final InfoStream infoStream
-
stallStartNS
private long stallStartNS
-
-
Constructor Detail
-
DocumentsWriterFlushControl
DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config)
-
-
Method Detail
-
activeBytes
public long activeBytes()
-
getFlushingBytes
long getFlushingBytes()
-
netBytes
long netBytes()
-
stallLimitBytes
private long stallLimitBytes()
-
assertMemory
private boolean assertMemory()
-
updatePeaks
private boolean updatePeaks(long delta)
-
ramBufferGranularity
private long ramBufferGranularity()
Return the smallest number of bytes that we would like to make sure to not miss from the global RAM accounting.
-
doAfterDocument
DocumentsWriterPerThread doAfterDocument(DocumentsWriterPerThread perThread)
-
checkout
private DocumentsWriterPerThread checkout(DocumentsWriterPerThread perThread, boolean markPending)
-
assertNumDocsSinceStalled
private boolean assertNumDocsSinceStalled(boolean stalled)
-
doAfterFlush
void doAfterFlush(DocumentsWriterPerThread dwpt)
-
updateStallState
private boolean updateStallState()
-
waitForFlush
public void waitForFlush()
-
setFlushPending
public void setFlushPending(DocumentsWriterPerThread perThread)
Sets flush pending state on the givenDocumentsWriterPerThread
. TheDocumentsWriterPerThread
must have indexed at least on Document and must not be already pending.
-
doOnAbort
void doOnAbort(DocumentsWriterPerThread perThread)
-
checkoutAndBlock
private void checkoutAndBlock(DocumentsWriterPerThread perThread)
To be called only by the owner of this object's monitor lock
-
checkOutForFlush
private DocumentsWriterPerThread checkOutForFlush(DocumentsWriterPerThread perThread)
-
addFlushingDWPT
private void addFlushingDWPT(DocumentsWriterPerThread perThread)
-
toString
public java.lang.String toString()
- Overrides:
toString
in classjava.lang.Object
-
nextPendingFlush
DocumentsWriterPerThread nextPendingFlush()
-
close
public void close()
- Specified by:
close
in interfacejava.lang.AutoCloseable
- Specified by:
close
in interfacejava.io.Closeable
-
allActiveWriters
public java.util.Iterator<DocumentsWriterPerThread> allActiveWriters()
Returns an iterator that provides access to all currently activeDocumentsWriterPerThread
s
-
doOnDelete
void doOnDelete()
-
getDeleteBytesUsed
public long getDeleteBytesUsed()
Returns heap bytes currently consumed by buffered deletes/updates that would be freed if we pushed all deletes. This does not include bytes consumed by already pushed delete/update packets.
-
ramBytesUsed
public long ramBytesUsed()
Description copied from interface:Accountable
Return the memory usage of this object in bytes. Negative values are illegal.- Specified by:
ramBytesUsed
in interfaceAccountable
-
numFlushingDWPT
int numFlushingDWPT()
-
getAndResetApplyAllDeletes
public boolean getAndResetApplyAllDeletes()
-
setApplyAllDeletes
public void setApplyAllDeletes()
-
obtainAndLock
DocumentsWriterPerThread obtainAndLock()
-
markForFullFlush
long markForFullFlush()
-
assertActiveDeleteQueue
private boolean assertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)
-
pruneBlockedQueue
private void pruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
Prunes the blockedQueue by removing all DWPTs that are associated with the given flush queue.
-
finishFullFlush
void finishFullFlush()
-
assertBlockedFlushes
boolean assertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)
-
abortFullFlushes
void abortFullFlushes()
-
abortPendingFlushes
void abortPendingFlushes()
-
isFullFlush
boolean isFullFlush()
Returnstrue
if a full flush is currently running
-
numQueuedFlushes
int numQueuedFlushes()
Returns the number of flushes that are already checked out but not yet actively flushing
-
numBlockedFlushes
int numBlockedFlushes()
Returns the number of flushes that are checked out but not yet available for flushing. This only applies during a full flush if a DWPT needs flushing but must not be flushed until the full flush has finished.
-
waitIfStalled
void waitIfStalled()
This method will block if too many DWPT are currently flushing and no checked out DWPT are available
-
anyStalledThreads
boolean anyStalledThreads()
Returnstrue
iff stalled
-
getInfoStream
public InfoStream getInfoStream()
Returns theIndexWriter
InfoStream
-
findLargestNonPendingWriter
DocumentsWriterPerThread findLargestNonPendingWriter()
-
checkoutLargestNonPendingWriter
final DocumentsWriterPerThread checkoutLargestNonPendingWriter()
Returns the largest non-pending flushable DWPT ornull
if there is none.
-
getPeakActiveBytes
long getPeakActiveBytes()
-
getPeakNetBytes
long getPeakNetBytes()
-
-