IBM Support

Server Crashes Due To Excessive InsertPermutations() Recursion

Technote (troubleshooting)


Problem

Preventing excessive InsertPermutations() recursion that can lead to stack overflow

Symptom

The server crashes


Cause

This has been identified as a product defect under APAR LO55991/SPR KHAN87ZUTS.

Diagnosing the problem

This crash will usually produce a core dump of the task. In this example, it is HTTP. When reviewing the NSD, you will find a crash stack that resembles the following stack:

Segmentation fault in

newLkLock__FP16DBCONTEXT_STRUCTP12LK_ID_STRUCTUlT3P7LK_INFOPc at

0xd16706bc ($t34)

0xd16706bc

(newLkLock__FP16DBCONTEXT_STRUCTP12LK_ID_STRUCTUlT3P7LK_INFOPc+0x44)

9421fb30 stwu r1,-1232(r1)

newLkLock__FP16DBCONTEXT_STRUCTP12LK_ID_STRUCTUlT3P7LK_INFOPc(0xa6ad8088

, 0x33eb6710, 0x4024, 0xffffffff, 0x33eb66f8, 0xd35448cc) at 0xd16706bc

_LkLock(0xa6ad8088, 0x33eb6710, 0x4024, 0xffffffff, 0x33eb66f8,

0xd35448cc) at 0xd1670160

LockDbWriteSemCtxTimedInt(0xa6ad8088, 0xffffffff, 0xd35448cc) at

0xd1523e80

LockDbRWCondInt(0xa6ad8088, 0x3c003c, 0x1, 0xd35448cc) at 0xd151dafc

LockDbRWInt(0xa6ad8088, 0x3c003c, 0xd35448cc) at 0xd151df44

AllocNSFPage(0xa6ad8088, 0x595d7d00, 0x131, 0x2000, 0x0, 0x595d7fe8,

0x575aa634, 0x33eb9384) at 0xd1603368

STMNewPageNSF(0xa6ad8088, 0x595d7d00, 0xd070d07, 0x2000, 0x0,

0x33eb8850, 0x33eb9384) at 0xd1607d10

BUFAllocAndPinNewPage(0xf0bfb1c8, 0xa6ad8088, 0x595d7d00, 0xd070d07,

0x2000, 0x0, 0x33eb939c, 0x33eb9384) at 0xd1971750

BUFAllocSpace(0xa6ad8088, 0x12a005, 0xf0bfb1c8, 0x16a016a0, 0x0,

0xffffffff, 0x33eb939c, 0x33eb9384) at 0xd19744fc

iNSFSlotAlloc(0xa6ad8088, 0x12a005, 0xf0bfb1c8, 0x169a169a, 0x0,

0xffffffff, 0x33eb9958) at 0xd1970a24

NSFSlotAllocCtx(0xa6ad8088, 0x12a005, 0xf0bfb1c8, 0x169a169a, 0x0,

0xffffffff, 0x33eb9958) at 0xd1967db8

BTStoreLongData(0xa6ad8088, 0x0, 0xa3e42a48, 0xf0bfb1c8, 0xa768c7e8,

0x100010, 0xa768c7f8, 0x169a169a) at 0xd1ddb178

BTStoreData(0xa6ad8088, 0x0, 0xa3e42a48, 0xf0bfb1c8, 0x33ebb934,

0xa768c7e8, 0x33ebb914, 0x33ebb918) at 0xd1df5808

BTStoreNode(0xa6ad8088, 0x0, 0xa3e42a48, 0xf0bfb1c8, 0x0, 0x33ebb934,

0xa768c7e8, 0x100010) at 0xd1e00388

BTKeyInsertExtCtx(0xa3e42a48, 0xa6ad8088, 0xf0bfb1c8, 0xa768c7e8,

0x100010, 0x16aa16aa, 0x4, 0x0) at 0xd1dca4f4

InsertEntry(0xa6ad7ccc) at 0xd1dbdd64

InsertResponse(0xa6ad7ccc, 0x33ebc040) at 0xd1f7a130

InsertPermutations(0xa5e707e8, 0xa6ad7ccc) at 0xd1f7ecc4

AddToIndex(0xa6ad7ccc, 0x33ebc218) at 0xd1d77260

InsertEntry, InsertResponse, InsertPermutations, AddtoIndex repeats for

some 800 lines in this example

InsertResponse(0xa6ad7ccc, 0x33eea440) at 0xd1f7a484

InsertPermutations(0xa5e707e8, 0xa6ad7ccc) at 0xd1f7ecc4

AddToIndex(0xa6ad7ccc, 0x33eea5f8) at 0xd1d77260

ProcessNote(0xa6ad7ccc, 0xa6afd254, 0x7394e7e8) at 0xd1fb49bc

UpdateCollection(0xa6afd24c, 0x0) at 0xd1fa7f5c

NIFUpdateCollectionNext(0x7ae07ae, 0xa6afd24c) at 0xd1fabb9c

NIFUpdateCollection(0x7ae07ae) at 0xd23800f4

NIFGetCollectionUpdated(0xa6ad7ccc, 0x1, 0x33eeb36c) at 0xd1d635a0

NIFOpenCollectionExtended4(0x3050c5c, 0x3050c5c, 0x13446, 0x0, 0x0,

0x0, 0x0, 0x0) at 0xd1d693a0

NIFOpenCollectionExtended3(0xc5c0c5c, 0xc5c0c5c, 0x13446, 0x0, 0x0,

0x0, 0x0, 0x0) at 0xd1d6a8c8

NIFOpenCollectionExtended2(0xc5c0c5c, 0xc5c0c5c, 0x13446, 0x0, 0x0,

0x0, 0x33eec6a6, 0x0) at 0xd1d6a9c4

NIFOpenCollection(0xc5c0c5c, 0xc5c0c5c, 0x13446, 0x0, 0x0, 0x33eec6a6,

0x0, 0x0) at 0xd1d6ab5c

GetCollectionPosition__5HaikuFP5NNoteiPPvPUlPUsT4(0x33eeccc8,

0x33ef3310, 0x5, 0x33eee170, 0x33eee300, 0x33eee0f0, 0x33eee10c) at

0xd50db174

Dispatch__Q2_5Haiku14AtFuncDispatchFP5HaikuP5NNoteiPPvPUlPUsT5(0xf0e267b

8, 0x33eeccc8, 0x33ef3310, 0x5, 0x33eee170, 0x33eee300, 0x33eee0f0,

0x33eee10c) at 0xd50f8d44

GetHaikuDatum__5HaikuFP5NNoteiPPvPUlPUsT4@AF1591_452(0x33eeccc8,

0x33ef3310, 0x5, 0x33eee170, 0x33eee300, 0x33eee0f0, 0x33eee10c,

0x5450545) at 0xd5127910

GetHaikuDatum__5HaikuFP5NNoteiPPvPUlPUsT4(0x33eeccc8, 0x33ef3310, 0x5,

0x33eee170, 0x33eee300, 0x33eee0f0, 0x33eee10c) at 0xd50f8b78

ExtensionProc__8NFormulaFUsT1PUlPPvPUsT3(0x33ef01b0, 0xb900b9, 0x50005,

0x33eee300, 0x33eee170, 0x33eee0f0, 0x33eee10c) at 0xd5140bfc

INotesCompExtProc(0x3504ff5c, 0x33ef01b0, 0xb900b9, 0x50005,

0x33eee300, 0x33eee170, 0x33eee0f0, 0x33eee100) at 0xd513baac

ExtensionProc__18CompGeneralContextFR7ComputeUlPP9CompValueT2(0x3504d75c

, 0x3504ff5c, 0xb9, 0x33eee590, 0x5) at 0xd16b0e50

Execute__13ExtensionProcFv(0x3505035c) at 0xd307ae44

ComputeVariants__14AtFunctionNodeFv(0x3505035c) at 0xd16bf070

ComputeVariants__14AtFunctionNodeFv(0x350503cc) at 0xd16bf034

ComputeVariants__4AtIfFv(0x35050438) at 0xd30b2a48

ComputeVariants__18MainExpressionNodeFv(0x35050100) at 0xd16c0ebc

ComputeVariants__8RootNodeFv(0x35050474) at 0xd16c1374

Eval__7ComputeFv(0x3504ff5c) at 0xd16b4bb4

NSFComputeEvaluateExt(0x3504ff5c, 0xf900f9, 0x33ef1430, 0x33eefd14,

0x1, 0x0, 0x0, 0x0) at 0xd16a8498

Evaluate__8NFormulaFR6NValuePi(0x33ef01b0, 0x33ef13b0, 0x0) at

0xd513c82c

AddField__9HuDocNoteFP7HuFieldPciPCc(0x33ef3310, 0x3504cf5c,

0x50e5e284, 0x1, 0x0) at 0xd53e2630

Write__10ShNotesVarFP17IFormOutputStream(0x34fbe040, 0x33ef33e8) at

0xd51cd76c

Write__12ShDictionaryFP17IFormOutputStream(0x351bcfbc, 0x33ef33e8) at

0xd51c72f0

Load__12ShDictionaryFP17IFormOutputStream(0x351bcfbc, 0x33ef33e8) at

0xd51c71a8

LoadItems__8HuLayoutFP17IFormOutputStream(0x35208230, 0x33ef33e8) at

0xd51c19cc

LoadSubForm__9HuDocNoteFPUci(0x33ef3310, 0x33ef17e8, 0x0) at 0xd53f1c78

LoadSubForm__9HuDocNoteFRC26HuComputedTextParseContexti(0x33ef3310,

0x350594a8, 0x0) at 0xd53f1e70

Load__15ShInsertSubFormFP17IFormOutputStream(0x35059380, 0x33ef33e8) at

0xd51c7fd4

LoadItems__8HuLayoutFP17IFormOutputStream(0x30058b94, 0x33ef33e8) at

0xd51c19cc

LoadSubForm__9HuDocNoteFPUci(0x33ef3310, 0x33ef1a88, 0x0) at 0xd53f1c78

LoadSubForm__9HuDocNoteFRC26HuComputedTextParseContexti(0x33ef3310,

0x30059578, 0x0) at 0xd53f1e70

Load__15ShInsertSubFormFP17IFormOutputStream(0x30059450, 0x33ef33e8) at

0xd51c7fd4

LoadItems__8HuLayoutFP17IFormOutputStream(0x3501f588, 0x33ef33e8) at

0xd51c19cc

LoadSubForm__9HuDocNoteFPUci(0x33ef3310, 0x350202f8, 0x0) at 0xd53f1c78

Load__15ShInsertSubFormFP17IFormOutputStream(0x350202d0, 0x33ef33e8) at

0xd51c8004

LoadItems__8HuLayoutFP17IFormOutputStream(0x3005a9b4, 0x33ef33e8) at

0xd51c19cc

LoadSubForm__9HuDocNoteFPUci(0x33ef3310, 0x33ef1e28, 0x0) at 0xd53f1c78

LoadSubForm__9HuDocNoteFRC26HuComputedTextParseContexti(0x33ef3310,

0x3005e670, 0x0) at 0xd53f1e70

Load__15ShInsertSubFormFP17IFormOutputStream(0x3005e548, 0x33ef33e8) at

0xd51c7fd4

LoadItems__8HuLayoutFP17IFormOutputStream(0x35000478, 0x33ef33e8) at

0xd51c19cc

LoadItemsAndHtml__9HuDocNoteFR8HuLayout(0x33ef3310, 0x35000478) at

0xd53ddac4

LoadItemsAndHtml__9HuDocNoteFv(0x33ef3310) at 0xd53dc684

LoadUsing__9HuDocNoteFR6HuForm(0x33ef3310, 0x35000478) at 0xd53dc770

HandleDominoCmd__5HaikuFR15OpenDocumentCmd(0x33ef3308, 0x36117028) at

0xd5764c14

HandleCmd__5HaikuFP3CmdR14CmdHandlerBase(0x36117028, 0x300504ec) at

0xd50f4498

PrivHandle__14CmdHandlerBaseFP3CmdT1(0x300504ec, 0x36117028, 0x0) at

0xd5266d70

PrivHandle__10CmdHandlerFP3Cmd(0x300504ec, 0x36117028) at 0xd526bc1c

Handler__10CmdHandlerFP3CmdPv(0x36117028, 0x300504ec) at 0xd526c3d0

Execute__3CmdFv(0x36117028) at 0xd5053118

InotesHTTPProcessRequestImpl__FP18_InotesHTTPrequest(0x3713d348) at

0xd50283f8

InotesHTTPProcessRequest(0x3713d348) at 0xd5027bd8

ProcessRequest__15HTInotesRequestFv(0x3713d33c) at 0xd4bf6a20

+FUHRER, GIL -LO0001PRM -L328/-------P3S3-12/06/01-07:40--AT

ProcessRequest__21HTRequestExtContainerF19HTApplicationSpaces(0x3713d058

, 0x5) at 0xd4bc7cd0

ProcessRequest__9HTRequestFv(0x3713cef0) at 0xd4bd3a1c

StartRequest__9HTSessionFv(0x35fa8820) at 0xd4be27e8

CheckForWork__14HTWorkerThreadFv(0x32714690) at 0xd4beee50

ThreadMain__14HTWorkerThreadFv(0x32714690) at 0xd4bee7f8

HTThreadBeginProc(0x32714690) at 0xd4b9d1e4

ThreadWrapper(0x0) at 0xd12e3d84


Resolving the problem

This issue is fixed in Domino version 8.5.2FP4 and above.

In those versions a notes.ini parameter, MAX_PERMUTE_RECURSE.

The initial setting can be 200 to start with, and lowered in increments of 50 from there until 50. The floor is 32. 200 has been shown to be sufficient to prevent the crash in the field in the past.



Document information

More support for: IBM Domino
Crash

Software version: 8.5.2.2

Operating system(s): AIX

Reference #: 1600317

Modified date: 07 August 2012