cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Aditya Bharadwaj (JIRA)" <j...@apache.org>
Subject [jira] [Comment Edited] (CASSANDRA-14136) MemtableFlushWriter DecoratedKey Exception
Date Wed, 03 Jan 2018 20:59:00 GMT

    [ https://issues.apache.org/jira/browse/CASSANDRA-14136?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16305216#comment-16305216
] 

Aditya Bharadwaj edited comment on CASSANDRA-14136 at 1/3/18 8:58 PM:
----------------------------------------------------------------------

Looks related. Seen this issue 38 times in the last 1 month, in all but 1 scenario, it seems
to have come while writing to a secondary index.
This is the exception

bq. java.lang.RuntimeException: Last written key DecoratedKey(-6638873113115166967, 81e77da2723b483d8c0d49f800c1e288)
>= current key DecoratedKey(-8794293631676762023, 9130cbbaa8e911e79641aba7018ec35280ee0000000800000000000007b6000000080000000000005bcc00000006d8add8acd9850000000ad983d8a8d98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000200000001000400026172000000000000343600000000007e4d080000000000001e06000000000000000182ed830ffc0590395ad535f900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bce080cd985d8aad988d8b3d8b7d8a9083dcccccd01081091408324a8e911e7ae1b578dff8303d380f00000000800000000000007b6000000080000000000005bce00000006d8add8acd9850000000cd985d8aad988d8b3d8b7d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000300000001000400026172000000000000343600000000007e4d080000000000001e08000000000000000182e98314fc0590395c213df900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bd0080ad8b5d8bad98ad8b1d8a9083dcccccd010810914d7b73a8e911e7ae1b578dff8303d380ee0000000800000000000007b6000000080000000000005bd000000006d8add8acd9850000000ad8b5d8bad98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000400000001000400026172000000000000343600000000007e4d3a0000000000001e22000000000000000183148310fc0889665cb9adf90400050030080850697a7a61487574080632303732363408046e756c6c0c080543727573740c0c0c0c080a435530303231353738320c0c0c0c080000000000009c400c080000000000000000080000000000000001084099999a0c0800000000000000000812d985d8a7d8b1d8acd8a7d8b1d98ad8aad8a70801410800000000007e4d3a0c0c0c08104d6172676865726974612043727573740c080c4f6d6e4372742d4d726774610c0815d986d988d8b920d8a7d984d8b9d8acd9)
writing into /mnt/DATA/cassandra/data/product/productdetails_by_storeid_variants-ec590ad0108611e7a92033b648576005/mc-25-big-Data.db

Even in this scenario, this table has a secondary index though.

{quote}CREATE TABLE prduct.productdetails_by_storeid_variants (
    merchantid uuid,
    languagecode text,
    storeid bigint,
    productid bigint,
    variantproductid bigint,
    quantity bigint,
    brand text,
    brandid text,
    bundlegroups text,
    catalogcode text,
    deliverymode text,
    deliverytime text,
    h1catname text,
    h2catname text,
    h3catname text,
    h4catname text,
    h5catname text,
    hierarchy1category text,
    hierarchy2category text,
    hierarchy3category text,
    hierarchy4category text,
    hierarchy5category text,
    image text,
    inventory bigint,
    largeimage text,
    longdescription text,
    maximumorderquantity bigint,
    minimumorderquantity bigint,
    mrp float,
    offerdesc text,
    primaryproductid bigint,
    producttitle text,
    producttype text,
    refid bigint,
    seodescription text,
    seokeywords text,
    seopagetitle text,
    seourlkey text,
    shortdescription text,
    sku text,
    smallimage text,
    tags text,
    variantproducts list<frozen<variant>>,
    variantproperty text,
    variantpropertyvalueid bigint,
    variantvalue text,
    webprice float,
    PRIMARY KEY (merchantid, languagecode, storeid, productid, variantproductid, quantity)
) WITH CLUSTERING ORDER BY (languagecode ASC, storeid ASC, productid ASC, variantproductid
ASC, quantity ASC)
    AND bloom_filter_fp_chance = 0.01
    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
    AND comment = ''
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND crc_check_chance = 1.0
    AND dclocal_read_repair_chance = 0.1
    AND default_time_to_live = 0
    AND gc_grace_seconds = 86400
    AND max_index_interval = 2048
    AND memtable_flush_period_in_ms = 0
    AND min_index_interval = 128
    AND read_repair_chance = 0.0
    AND speculative_retry = '99PERCENTILE';
CREATE INDEX productdetails_by_storeid_variants_refid_idx ON product.productdetails_by_storeid_variants
(refid);{quote}


was (Author: adityabharadwaj):
Looks related. Seen this issue 38 times in the last 1 month, in all but 1 scenario, it seems
to have come while writing to a secondary index.
This is the exception

bq. java.lang.RuntimeException: Last written key DecoratedKey(-6638873113115166967, 81e77da2723b483d8c0d49f800c1e288)
>= current key DecoratedKey(-8794293631676762023, 9130cbbaa8e911e79641aba7018ec35280ee0000000800000000000007b6000000080000000000005bcc00000006d8add8acd9850000000ad983d8a8d98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000200000001000400026172000000000000343600000000007e4d080000000000001e06000000000000000182ed830ffc0590395ad535f900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bce080cd985d8aad988d8b3d8b7d8a9083dcccccd01081091408324a8e911e7ae1b578dff8303d380f00000000800000000000007b6000000080000000000005bce00000006d8add8acd9850000000cd985d8aad988d8b3d8b7d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000300000001000400026172000000000000343600000000007e4d080000000000001e08000000000000000182e98314fc0590395c213df900f0010034080850697a7a6148757408063230373236340c0808546f7070696e677308065361756365730c0c0c080a43553030323135373638080a435530303231353737380c0c0c0800000000000003e80880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c080000000000000000080000000000000001083dcccccd0c0800000000000000000817d8b5d984d8b5d8a920d8a7d984d8a8d98ad8aad8b2d8a70801410800000000007e4d080c080e4f6d6e2d537563732d507a5375630880852f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f536f757263652f69636f6e5f73617563655f4242512e706e670c0806d8add8acd985080000000000005bd0080ad8b5d8bad98ad8b1d8a9083dcccccd010810914d7b73a8e911e7ae1b578dff8303d380ee0000000800000000000007b6000000080000000000005bd000000006d8add8acd9850000000ad8b5d8bad98ad8b1d8a900000000000000080000000000000001000000992f2f696d616765732d63646e2d79756d2e6d6172746a61636b2e636f6d2f617a7572652f79756d2d7265736f75726365732f38316537376461322d373233622d343833642d386330642d3439663830306331653238382f496d616765732f50726f64756374496d616765732f5377617463682f4c617267655f49636f6e2f69636f6e5f73617563655f4242512e706e673b77696474683d333600000008000000000000000400000001000400026172000000000000343600000000007e4d3a0000000000001e22000000000000000183148310fc0889665cb9adf90400050030080850697a7a61487574080632303732363408046e756c6c0c080543727573740c0c0c0c080a435530303231353738320c0c0c0c080000000000009c400c080000000000000000080000000000000001084099999a0c0800000000000000000812d985d8a7d8b1d8acd8a7d8b1d98ad8aad8a70801410800000000007e4d3a0c0c0c08104d6172676865726974612043727573740c080c4f6d6e4372742d4d726774610c0815d986d988d8b920d8a7d984d8b9d8acd9)
writing into /mnt/DATA/cassandra/data/martjack/productdetails_by_storeid_variants-ec590ad0108611e7a92033b648576005/mc-25-big-Data.db

Even in this scenario, this table has a secondary index though.

{quote}CREATE TABLE martjack.productdetails_by_storeid_variants (
    merchantid uuid,
    languagecode text,
    storeid bigint,
    productid bigint,
    variantproductid bigint,
    quantity bigint,
    brand text,
    brandid text,
    bundlegroups text,
    catalogcode text,
    deliverymode text,
    deliverytime text,
    h1catname text,
    h2catname text,
    h3catname text,
    h4catname text,
    h5catname text,
    hierarchy1category text,
    hierarchy2category text,
    hierarchy3category text,
    hierarchy4category text,
    hierarchy5category text,
    image text,
    inventory bigint,
    largeimage text,
    longdescription text,
    maximumorderquantity bigint,
    minimumorderquantity bigint,
    mrp float,
    offerdesc text,
    primaryproductid bigint,
    producttitle text,
    producttype text,
    refid bigint,
    seodescription text,
    seokeywords text,
    seopagetitle text,
    seourlkey text,
    shortdescription text,
    sku text,
    smallimage text,
    tags text,
    variantproducts list<frozen<variant>>,
    variantproperty text,
    variantpropertyvalueid bigint,
    variantvalue text,
    webprice float,
    PRIMARY KEY (merchantid, languagecode, storeid, productid, variantproductid, quantity)
) WITH CLUSTERING ORDER BY (languagecode ASC, storeid ASC, productid ASC, variantproductid
ASC, quantity ASC)
    AND bloom_filter_fp_chance = 0.01
    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
    AND comment = ''
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND crc_check_chance = 1.0
    AND dclocal_read_repair_chance = 0.1
    AND default_time_to_live = 0
    AND gc_grace_seconds = 86400
    AND max_index_interval = 2048
    AND memtable_flush_period_in_ms = 0
    AND min_index_interval = 128
    AND read_repair_chance = 0.0
    AND speculative_retry = '99PERCENTILE';
CREATE INDEX productdetails_by_storeid_variants_refid_idx ON martjack.productdetails_by_storeid_variants
(refid);{quote}

> MemtableFlushWriter DecoratedKey Exception
> ------------------------------------------
>
>                 Key: CASSANDRA-14136
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-14136
>             Project: Cassandra
>          Issue Type: Bug
>          Components: Core, Secondary Indexes
>         Environment: CentOS Linux release 7.3.1611 (Core) , 3 Node cluster with Replication
Factor or 2
>            Reporter: Aditya Bharadwaj
>         Attachments: debug.log.gz, system.log.gz
>
>
> Running into this issue on my cluster periodically for different tables. After this error
is encountered, all the post flushes stop and eventually the system runs out of memory.
> On a restart all the commit logs get played normally and things go back to normal. 
> I'm unable to understand the scenario, but the issue is recreating every few days.
> {code}DEBUG [MemtableFlushWriter:884] 2017-12-26 18:19:40,883 Memtable.java:401 - Completed
flushing /mnt/DATA/cassandra/data/products/products_by_hierarchy5storeid_pascdesc-411cabe0632411e7b25a1b665c06298b/.id
> x_hierarchy1category/mc-2050-big-Data.db (508.127KiB) for commitlog position ReplayPosition(segmentId=1513929386900,
>  position=19110822)
> DEBUG [MemtableFlushWriter:884] 2017-12-26 18:19:41,150 Memtable.java:368 - Writing Memtable-products_by_hierarchy5storeid_pascdesc.idx_hierarchy3category@551487729(545.926KiB
serialized bytes, 324073 ops, 0%/0% of on/off-heap limit)
> ERROR [MemtableFlushWriter:884] 2017-12-26 18:19:41,316 CassandraDaemon.java:205 - Exception
in thread Thread[MemtableFlushWriter:884,5,main]
> java.lang.RuntimeException: Last written key DecoratedKey(CU00328612, 43553030333238363132)
>= current key DecoratedKey(^@^@^@^@^@^@^@^@^@^@, 43553030333238363838) writing into /mnt/DATA/cassandra/data/products/products_by_hierarchy5storeid_pascdesc-411cabe0632411e7b25a1b665c06298b/.idx_hierarchy3category/mc-2134-big-Data.db
>         at org.apache.cassandra.io.sstable.format.big.BigTableWriter.beforeAppend(BigTableWriter.java:106)
~[apache-cassandra-3.0.9.jar:3.0.9]
>         at org.apache.cassandra.io.sstable.format.big.BigTableWriter.append(BigTableWriter.java:145)
~[apache-cassandra-3.0.9.jar:3.0.9]
>         at org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter.append(SimpleSSTableMultiWriter.java:45)
~[apache-cassandra-3.0.9.jar:3.0.9]
>         at org.apache.cassandra.io.sstable.SSTableTxnWriter.append(SSTableTxnWriter.java:52)
~[apache-cassandra-3.0.9.jar:3.0.9]
>         at org.apache.cassandra.db.Memtable.writeSortedContents(Memtable.java:394) ~[apache-cassandra-3.0.9.jar:3.0.9]
>         at org.apache.cassandra.db.Memtable.flush(Memtable.java:332) ~[apache-cassandra-3.0.9.jar:3.0.9]
>         at org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1054)
~[apache-cassandra-3.0.9.jar:3.0.9]
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[na:1.8.0_112]
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
~[na:1.8.0_112]
>         at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_112]
> {code}



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message