/*
 * JBoss, Home of Professional Open Source.
 * Copyright 2014, Red Hat, Inc., and individual contributors
 * as indicated by the @author tags. See the copyright.txt file in the
 * distribution for a full listing of individual contributors.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */

package org.jboss.as.test.integration.domain.suites;

import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.ACTIVE_OPERATION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.CANCELLED;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.CHILD_TYPE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.CORE_SERVICE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.EXECUTION_STATUS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.EXTENSION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.FAILED;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.HOST;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.MANAGEMENT;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.MANAGEMENT_OPERATIONS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.PRIMARY;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.NAME;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.OP;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.OP_ADDR;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.OUTCOME;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.PROFILE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.READ_ATTRIBUTE_OPERATION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.READ_CHILDREN_RESOURCES_OPERATION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.REMOVE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SERVER;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SERVICE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SUBSYSTEM;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SUCCESS;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.BLOCK_POINT;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.CALLER;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.TARGET_HOST;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.TARGET_SERVER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import org.jboss.as.controller.OperationContext;
import org.jboss.as.controller.PathAddress;
import org.jboss.as.controller.PathElement;
import org.jboss.as.controller.client.ModelControllerClient;
import org.jboss.as.controller.client.OperationMessageHandler;
import org.jboss.as.controller.client.helpers.domain.DomainClient;
import org.jboss.as.controller.operations.common.Util;
import org.jboss.as.test.integration.domain.extension.ExtensionSetup;
import org.jboss.as.test.integration.domain.management.util.DomainTestSupport;
import org.jboss.as.test.integration.domain.management.util.DomainTestUtils;
import org.jboss.as.test.integration.management.extension.blocker.BlockerExtension;
import org.jboss.as.test.integration.management.util.MgmtOperationException;
import org.jboss.as.test.shared.TimeoutUtil;
import org.jboss.dmr.ModelNode;
import org.jboss.dmr.ModelType;
import org.jboss.dmr.Property;
import org.jboss.logging.Logger;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;

/**
 * Test of cancellation of running operations in a domain.
 *
 * @author Brian Stansberry (c) 2014 Red Hat Inc.
 */
public class OperationCancellationTestCase {

    private static final Logger log = Logger.getLogger(OperationCancellationTestCase.class);

    private static final PathElement SUBSYSTEM_ELEMENT = PathElement.pathElement(SUBSYSTEM, BlockerExtension.SUBSYSTEM_NAME);
    private static final PathAddress SUBSYSTEM_ADDRESS = PathAddress.pathAddress(
            PathElement.pathElement(PROFILE, "default"),
            SUBSYSTEM_ELEMENT);
    private static final ModelNode BLOCK_OP = Util.createEmptyOperation("block", SUBSYSTEM_ADDRESS);
    private static final ModelNode WRITE_FOO_OP = Util.getWriteAttributeOperation(SUBSYSTEM_ADDRESS, BlockerExtension.FOO.getName(), true);
    private static final PathAddress MGMT_CONTROLLER = PathAddress.pathAddress(
            PathElement.pathElement(CORE_SERVICE, MANAGEMENT),
            PathElement.pathElement(SERVICE, MANAGEMENT_OPERATIONS)
    );
    private static final PathAddress PRIMARY_ADDRESS = PathAddress.pathAddress(HOST, PRIMARY);
    private static final PathAddress SECONDARY_ADDRESS = PathAddress.pathAddress(HOST, "secondary");

    private static final Set<BlockerExtension.BlockPoint> RUNTIME_POINTS =
            EnumSet.of(BlockerExtension.BlockPoint.RUNTIME, BlockerExtension.BlockPoint.SERVICE_START, BlockerExtension.BlockPoint.SERVICE_STOP);
    private static final long GET_TIMEOUT = TimeoutUtil.adjust(10000);

    private static DomainTestSupport testSupport;
    private static DomainClient primaryClient;

    @BeforeClass
    public static void setupDomain() throws Exception {
        testSupport = DomainTestSuite.createSupport(OperationCancellationTestCase.class.getSimpleName());
        primaryClient = testSupport.getDomainPrimaryLifecycleUtil().getDomainClient();

        // Initialize the test extension
        ExtensionSetup.initializeBlockerExtension(testSupport);

        PathAddress extensionAddress = PathAddress.pathAddress(EXTENSION, BlockerExtension.MODULE_NAME);
        ModelNode addExtension = Util.createAddOperation(extensionAddress);

        executeForResult(addExtension, primaryClient);

        ModelNode addSubsystem = Util.createAddOperation(PathAddress.pathAddress(
                PathElement.pathElement(PROFILE, "default"),
                PathElement.pathElement(SUBSYSTEM, BlockerExtension.SUBSYSTEM_NAME)));
        executeForResult(addSubsystem, primaryClient);

        ModelNode addPrimaryExtension = Util.createAddOperation(PRIMARY_ADDRESS.append(extensionAddress));
        executeForResult(addPrimaryExtension, primaryClient);

        ModelNode addPrimarySubsystem = Util.createAddOperation(PRIMARY_ADDRESS.append(SUBSYSTEM_ELEMENT));
        executeForResult(addPrimarySubsystem, primaryClient);

        ModelNode addSecondaryExtension = Util.createAddOperation(SECONDARY_ADDRESS.append(extensionAddress));
        executeForResult(addSecondaryExtension, primaryClient);

        ModelNode addSecondarySubsystem = Util.createAddOperation(SECONDARY_ADDRESS.append(SUBSYSTEM_ELEMENT));
        executeForResult(addSecondarySubsystem, primaryClient);
    }

    @AfterClass
    public static void tearDownDomain() throws Exception {
        ModelNode removeSubsystem = Util.createEmptyOperation(REMOVE, PathAddress.pathAddress(
                PathElement.pathElement(PROFILE, "default"),
                SUBSYSTEM_ELEMENT));
        executeForResult(removeSubsystem, primaryClient);

        PathAddress extensionAddress = PathAddress.pathAddress(EXTENSION, BlockerExtension.MODULE_NAME);
        ModelNode removeExtension = Util.createEmptyOperation(REMOVE, extensionAddress);
        executeForResult(removeExtension, primaryClient);

        ModelNode removeSecondarySubsystem = Util.createEmptyOperation(REMOVE, SECONDARY_ADDRESS.append(SUBSYSTEM_ELEMENT));
        executeForResult(removeSecondarySubsystem, primaryClient);

        ModelNode removeSecondaryExtension = Util.createEmptyOperation(REMOVE, SECONDARY_ADDRESS.append(extensionAddress));
        executeForResult(removeSecondaryExtension, primaryClient);

        ModelNode removePrimarySubsystem = Util.createEmptyOperation(REMOVE, PRIMARY_ADDRESS.append(SUBSYSTEM_ELEMENT));
        executeForResult(removePrimarySubsystem, primaryClient);

        ModelNode removePrimaryExtension = Util.createEmptyOperation(REMOVE, PRIMARY_ADDRESS.append(extensionAddress));
        executeForResult(removePrimaryExtension, primaryClient);

        testSupport = null;
        primaryClient = null;
        DomainTestSuite.stopSupport();
    }

    @After
    public void awaitCompletion() throws Exception {
        // Start from the leaves of the domain process tree and work inward validating
        // that all block ops are cleared locally. This ensures that a later test doesn't
        // mistakenly cancel a completing op from an earlier test
        validateNoActiveOperation(primaryClient, "primary", "main-one");
        validateNoActiveOperation(primaryClient, "secondary", "main-three");
        validateNoActiveOperation(primaryClient, "secondary", null);
        validateNoActiveOperation(primaryClient, "primary", null);
    }

    @Test
    public void testPrimaryBlockModel() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.EXECUTING, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.EXECUTING, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertFailedOrCancelled(response);
        validateNoActiveOperation(primaryClient, "primary", null, id, false);
    }

    @Test
    public void testPrimaryBlockRuntime() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.EXECUTING, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.EXECUTING, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertFailedOrCancelled(response);
        validateNoActiveOperation(primaryClient, "primary", null, id, false);
    }

    @Test
    @Ignore("MSC-143")
    public void testPrimaryBlockServiceStart() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", null, id, false);
    }

    @Test
    @Ignore("MSC-143")
    public void testPrimaryBlockServiceStartCancelFuture() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        blockFuture.cancel(true);
        try {
            ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
            fail("not cancelled: " + response);
        } catch (CancellationException good) {
            // good
        }
        validateNoActiveOperation(primaryClient, "primary", null, id, false);
    }

    @Test
    public void testPrimaryBlockVerify() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.EXECUTING, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.EXECUTING, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertFailedOrCancelled(response);
        validateNoActiveOperation(primaryClient, "primary", null, id, false);
    }

    @Test
    public void testPrimaryBlockCompletion() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.COMMIT);
        String blockId = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), SUCCESS, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", null, blockId, false);
    }

    @Test
    public void testPrimaryBlockRollback() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.ROLLBACK);
        String blockId = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.ROLLING_BACK, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // primary has already failed when we cancel, so it reports as failed
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", null, blockId, false);
    }

    @Test
    public void testPrimaryBlockOtherOpCancelOtherOp() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", null, BlockerExtension.BlockPoint.COMMIT);
        String blockId = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start);
        Future<ModelNode> writeFuture = primaryClient.executeAsync(WRITE_FOO_OP, OperationMessageHandler.DISCARD);
        cancel(primaryClient, "primary", null, "write-attribute", OperationContext.ExecutionStatus.AWAITING_OTHER_OPERATION, start, false);
        ModelNode response = writeFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        blockFuture.cancel(true);
        validateNoActiveOperation(primaryClient, "primary", null, blockId, true);
    }

    @Test
    public void testPrimaryBlockOtherOpCancelBlockOp() throws Exception {
        long start = System.currentTimeMillis();
        block("primary", null, BlockerExtension.BlockPoint.COMMIT);
        String blockId = findActiveOperation(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start);
        Future<ModelNode> writeFuture = primaryClient.executeAsync(WRITE_FOO_OP, OperationMessageHandler.DISCARD);
        findActiveOperation(primaryClient, "primary", null, "write-attribute", OperationContext.ExecutionStatus.AWAITING_OTHER_OPERATION, start);
        cancel(primaryClient, "primary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start, false);
        ModelNode response = writeFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), SUCCESS, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", null, blockId, true);
    }

    @Test
    public void testSecondaryBlockModelCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    public void testSecondaryBlockRuntimeCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryBlockServiceStartCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryBlockServiceStartCancelPrimaryFuture() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        blockFuture.cancel(true);
        try {
            ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
            fail("not cancelled: " + response);
        } catch (CancellationException good) {
            // good
        }
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    public void testSecondaryBlockVerifyCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    public void testSecondaryBlockCompletionCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.COMMIT);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // The primary may or may not be done with DomainSecondaryHandler.execute and DomainRolloutStepHandler.execute
        // when cancelled. If yes, result is SUCCESS, if not result is CANCELLED
        assertSuccessOrCancelled(response);
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    public void testSecondaryBlockRollbackCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.ROLLBACK);
        String id = findActiveOperation(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // Secondary fails before Stage.DONE so no normal prepared message in DONE with the failure. The block in
        // rollback prevents the failure coming via the final response. So the primary doesn't know about
        // the failure and just detects the local cancellation. So, CANCELLED
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, id, true);
    }

    @Test
    public void testSecondaryBlockOtherOpCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.MODEL);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        Future<ModelNode> writeFuture = primaryClient.executeAsync(WRITE_FOO_OP, OperationMessageHandler.DISCARD);
        cancel(primaryClient, "primary", null, "write-attribute", OperationContext.ExecutionStatus.AWAITING_OTHER_OPERATION, start, false);
        ModelNode response = writeFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        blockFuture.cancel(true);
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, true);
    }

    @Test
    public void testSecondaryBlockModelCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.MODEL);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "secondary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, false);
    }

    @Test
    public void testSecondaryBlockRuntimeCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.RUNTIME);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "secondary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, false);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryBlockServiceStartCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.SERVICE_START);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, false);
    }

    @Test
    public void testSecondaryBlockVerifyCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.VERIFY);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "secondary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, false);
    }

    @Test
    public void testSecondaryBlockCompletionCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.COMMIT);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start);
        cancel(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.COMPLETING , start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // cancellation any time in Stage.DONE on secondary will not result in a prepare-phase failed response to
        // primary, so this should always be success
        assertEquals(response.asString(), SUCCESS, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, true);
    }

    @Test
    public void testSecondaryBlockRollbackCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", null, BlockerExtension.BlockPoint.ROLLBACK);
        String blockId = findActiveOperation(primaryClient, "secondary", null, "block", null, start);
        cancel(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.ROLLING_BACK, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // Cancelling the rollback does not prevent the primary learning that the secondary failed.
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", null, blockId, true);
    }

    @Test
    public void testPrimaryServerBlockModelCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testPrimaryServerBlockRuntimeCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testPrimaryServerBlockServiceStartCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testPrimaryServerBlockServiceStartCancelPrimaryFuture() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        blockFuture.cancel(true);
        try {
            ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
            fail("not cancelled: " + response);
        } catch (CancellationException good) {
            // good
        }
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testPrimaryServerBlockVerifyCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testPrimaryServerBlockCompletionCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.COMMIT);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.COMPLETING, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // The result may be success or cancelled depending on whether the cancelled executed while
        // DomainRolloutStepHandler.execute was running (CANCELLED) or afterwards (SUCCESS)
        assertSuccessOrCancelled(response);
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testPrimaryServerBlockRollbackCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.ROLLBACK);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // The server op does not reach the prepare stage, so the primary doesn't either and reports as cancelled
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testPrimaryServerBlockModelCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", "main-one", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, false);
    }

    @Test
    public void testPrimaryServerBlockRuntimeCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", "main-one", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, false);
    }

    @Test
    @Ignore("MSC-143")
    public void testPrimaryServerBlockServiceStartCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, false);
    }

    @Test
    public void testPrimaryServerBlockVerifyCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", null, start);
        cancel(primaryClient, "primary", "main-one", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, false);
    }

    @Test
    public void testPrimaryServerBlockCompletionCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.COMMIT);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.COMPLETING, start);
        cancel(primaryClient, "primary", "main-one", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // cancelling on the server during Stage.DONE should not result in a prepare-phase failure sent to primary,
        // so result should always be SUCCESS
        assertEquals(response.asString(), SUCCESS, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testPrimaryServerBlockRollbackCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("primary", "main-one", BlockerExtension.BlockPoint.ROLLBACK);
        String id = findActiveOperation(primaryClient, "primary", "main-one", "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        cancel(primaryClient, "primary", "main-one", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // Server never reaches Stage.DONE before blocking, so primary is waiting for initial response.
        // Cancelling the rollback on server doesn't change that response from FAILED. So primary sees and reports failure.
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "primary", "main-one", id, true);
    }

    @Test
    public void testSecondaryServerBlockModelCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id , true);
    }

    @Test
    public void testSecondaryServerBlockRuntimeCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryServerBlockServiceStartCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryServerBlockServiceStartCancelPrimaryFuture() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        blockFuture.cancel(true);
        try {
            ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
            fail("not cancelled: " + response);
        } catch (CancellationException good) {
            // good
        }
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockVerifyCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockCompletionCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.COMMIT);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.COMPLETING, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // The result may be success or cancelled depending on whether the cancelled executed while
        // DomainRolloutStepHandler.execute was running (CANCELLED) or afterwards (SUCCESS)
        assertSuccessOrCancelled(response);
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockRollbackCancelPrimary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.ROLLBACK);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        cancel(primaryClient, "primary", null, "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // Server doesn't get to DONE so doesn't report there, so primary is waiting for initial report, which is blocking
        // on server. So primary detects local cancellation and reports it
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }


    // Tests of cancelling an op blocking on a secondary server via an op on the secondary

    // NOTE: Not the way a user should cancel a server op -- either cancel it on the coordinating HC
    // (i.e. primary for a domain-wide op) or on the server itself. Otherwise it's easy to mistakenly
    // cancel the primary's call to the secondary HC and not its proxied call to the server.
    // But we test this for completeness.

    @Test
    public void testSecondaryServerBlockModelCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        // Here we must pass 'true' to the 'serverOpOnly' param to ensure we cancel the server op, and not the non-blocking HC op
        cancel(primaryClient, "secondary", null, "block", null, start, true);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockRuntimeCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        // Here we must pass 'true' to the 'serverOpOnly' param to ensure we cancel the server op, and not the non-blocking HC op
        cancel(primaryClient, "secondary", null, "block", null, start, true);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryServerBlockServiceStartCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start);
        // Here we must pass 'true' to the 'serverOpOnly' param to ensure we cancel the server op, and not the non-blocking HC op
        cancel(primaryClient, "secondary", null, "block", null, start, true);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockVerifyCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        // Here we must pass 'true' to the 'serverOpOnly' param to ensure we cancel the server op, and not the non-blocking HC op
        cancel(primaryClient, "secondary", null, "block", null, start, true);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockCompletionCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.COMMIT);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.COMPLETING, start);
        // Here we must pass 'true' to the 'serverOpOnly' param to ensure we cancel the server op, and not the non-blocking HC op
        cancel(primaryClient, "secondary", null, "block", null, start, true);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // The secondary will already have sent its prepare phase response to primary before the secondary op even gets started.
        // So the subsequent cancellation will not affect the overall result, so it's SUCCESS
        assertEquals(response.asString(), SUCCESS, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockRollbackCancelSecondary() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.ROLLBACK);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        // Here we must pass 'true' to the 'serverOpOnly' param to ensure we cancel the server op, and not the non-blocking HC op
        cancel(primaryClient, "secondary", null, "block", null, start, true);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // Server can't report it's failure as a prepared message in DONE because it doesn't get there. So primary is
        // waiting for the initial report. The secondary then cancelling releases the initial report but doesn't change
        // its outcome from FAILED. So primary sees failure
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testSecondaryServerBlockModelCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.MODEL);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "secondary", "main-three", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, false);
    }

    @Test
    public void testSecondaryServerBlockRuntimeCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "secondary", "main-three", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, false);
    }

    @Test
    @Ignore("MSC-143")
    public void testSecondaryServerBlockServiceStartCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.SERVICE_START);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.AWAITING_STABILITY, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, false);
    }

    @Test
    public void testSecondaryServerBlockVerifyCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.VERIFY);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "secondary", "main-three", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, false);
    }

    @Test
    public void testSecondaryServerBlockCompletionCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.COMMIT);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", null, start);
        cancel(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.COMPLETING, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // cancelling on the server during Stage.DONE should not result in a prepare-phase failure sent to primary,
        // so result should always be SUCCESS
        assertEquals(response.asString(), SUCCESS, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, false);
    }

    @Test
    public void testSecondaryServerBlockRollbackCancelServer() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.ROLLBACK);
        String id = findActiveOperation(primaryClient, "secondary", "main-three", "block", OperationContext.ExecutionStatus.ROLLING_BACK, start);
        cancel(primaryClient, "secondary", "main-three", "block", null, start, false);
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        // Cancelling in rollback doesn't change the server's FAILED outcome, so primary sees it and reports it.
        assertEquals(response.asString(), FAILED, response.get(OUTCOME).asString());
        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, false);
    }

    // Tests of helper methods

    @Test
    public void testCancelNonProgressingOperation() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "primary", null, "block", null, start);
        ModelNode op = Util.createEmptyOperation("cancel-non-progressing-operation",
                PathAddress.pathAddress(PathElement.pathElement(HOST, "primary")).append(MGMT_CONTROLLER));
        op.get("timeout").set(0);
        ModelNode result = executeForResult(op, primaryClient);
        assertEquals(result.toString(), id, result.asString());
        ModelNode response = blockFuture.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
        assertEquals(response.asString(), CANCELLED, response.get(OUTCOME).asString());
        // Bogus as "id" is not the id on secondary/main-three
//        validateNoActiveOperation(primaryClient, "secondary", "main-three", id, true);
    }

    @Test
    public void testFindNonProgressingOperation() throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.RUNTIME);
        String id = findActiveOperation(primaryClient, "primary", null, "block", null, start);
        try {
            ModelNode op = Util.createEmptyOperation("find-non-progressing-operation",
                    PathAddress.pathAddress(PathElement.pathElement(HOST, "primary")).append(MGMT_CONTROLLER));
            op.get("timeout").set(0);
            ModelNode result = executeForResult(op, primaryClient);
            assertEquals(result.toString(), id, result.asString());
        } finally {
            try {
                blockFuture.cancel(true);
                validateNoActiveOperation(primaryClient, "primary", null, id, true);
            } catch (Exception toLog) {
                log.error("Failed to cancel in testFindNonProgressingOperation" , toLog);
            }
        }
    }

    /** WFCORE-263 */
    @Test
    public void testCancellingNonProgressingDomainRollout() throws Exception {
        failNonProgressingDomainRolloutTest("cancel-non-progressing-operation");
    }

    /** WFCORE-263 */
    @Test
    public void testFindNonProgressingDomainRollout() throws Exception {
        failNonProgressingDomainRolloutTest("find-non-progressing-operation");
    }

    private void failNonProgressingDomainRolloutTest(String opName) throws Exception {
        long start = System.currentTimeMillis();
        Future<ModelNode> blockFuture = block("secondary", "main-three", BlockerExtension.BlockPoint.RUNTIME);
        String id = null;
        try {
            id = findActiveOperation(primaryClient, "secondary", null, "block", OperationContext.ExecutionStatus.COMPLETING, start);
            ModelNode op = Util.createEmptyOperation(opName,
                    PathAddress.pathAddress(PathElement.pathElement(HOST, "secondary")).append(MGMT_CONTROLLER));
            op.get("timeout").set(0);
            ModelNode cancelled = executeForResult(op, primaryClient);
            assertTrue(cancelled.isDefined());
            if (opName.contains("cancel")) {
                validateNoActiveOperation(primaryClient, "secondary", null, cancelled.asString(), true);
            }
        } finally {
            try {
                blockFuture.cancel(true);
                if (id != null) {
                    validateNoActiveOperation(primaryClient, "secondary", null, id, true);
                }
            } catch (Exception toLog) {
                log.error("Failed to cancel in failNonProgressingDomainRolloutTest of " + opName , toLog);
            }
        }
    }

    private Future<ModelNode> block(String host, String server, BlockerExtension.BlockPoint blockPoint) {
        ModelNode op = BLOCK_OP.clone();
        op.get(TARGET_HOST.getName()).set(host);
        if (server != null) {
            op.get(TARGET_SERVER.getName()).set(server);
        }
        op.get(BLOCK_POINT.getName()).set(blockPoint.toString());
        op.get(CALLER.getName()).set(getTestMethod());
        if (server == null && RUNTIME_POINTS.contains(blockPoint)) {
            // retarget the op to the HC subsystem
            PathAddress addr = host.equals("primary") ? PRIMARY_ADDRESS : SECONDARY_ADDRESS;
            op.get(OP_ADDR).set(addr.append(SUBSYSTEM_ELEMENT).toModelNode());
        }
        return primaryClient.executeAsync(op, OperationMessageHandler.DISCARD);
    }

    private void cancel(DomainClient client, String host, String server, String opName,
                           OperationContext.ExecutionStatus targetStatus, long executionStart, boolean serverOpOnly) throws Exception {
        PathAddress address = PathAddress.pathAddress(PathElement.pathElement(HOST, host));
        if (server != null) {
            address = address.append(PathElement.pathElement(SERVER, server));
        }
        address = address.append(MGMT_CONTROLLER);

        final String opToCancel = findActiveOperation(client, address, opName, targetStatus, executionStart, serverOpOnly);
        address = address.append(PathElement.pathElement(ACTIVE_OPERATION, opToCancel));

        ModelNode result = executeForResult(Util.createEmptyOperation("cancel", address), client);

        boolean retVal = result.asBoolean();
        assertTrue(result.asString(), retVal);
    }

    private String findActiveOperation(DomainClient client, String host, String server, String opName,
                                       OperationContext.ExecutionStatus targetStatus, long executionStart) throws Exception {
        PathAddress address = getManagementControllerAddress(host, server);
        return findActiveOperation(client, address, opName, targetStatus, executionStart, false);
    }

    private String findActiveOperation(DomainClient client, PathAddress address, String opName, OperationContext.ExecutionStatus targetStatus, long executionStart, boolean serverOpOnly) throws Exception {
        ModelNode op = Util.createEmptyOperation(READ_CHILDREN_RESOURCES_OPERATION, address);
        op.get(CHILD_TYPE).set(ACTIVE_OPERATION);
        long maxTime = TimeoutUtil.adjust(5000);
        long timeout = executionStart + maxTime;
        List<String> activeOps = new ArrayList<String>();
        String opToCancel = null;
        do {
            activeOps.clear();
            ModelNode result = executeForResult(op, client);
            if (result.isDefined()) {
                assertEquals(result.asString(), ModelType.OBJECT, result.getType());
                for (Property prop : result.asPropertyList()) {
                    if (prop.getValue().get(OP).asString().equals(opName)) {
                        PathAddress pa = PathAddress.pathAddress(prop.getValue().get(OP_ADDR));
                        if (!serverOpOnly || pa.size() > 2 && pa.getElement(1).getKey().equals(SERVER)) {
                            if (targetStatus == null || prop.getValue().get(EXECUTION_STATUS).asString().equals(targetStatus.toString())) {
                                if (targetStatus != null && opToCancel == null) {
                                    // Ignore the fact there were other ops with a different status
                                    activeOps.clear();
                                }
                                opToCancel = prop.getName();
                            } else {
                                // Ignore the fact there were other ops with a different status
                                activeOps.clear();
                            }
                            activeOps.add(prop.getName() + " -- " + prop.getValue().toString());
                        }
                    }
                }
            }
            if (opToCancel == null) {
                Thread.sleep(50);
            }

        } while ((opToCancel == null || activeOps.size() > 1) && System.currentTimeMillis() <= timeout);

        assertTrue(opName + " not present after " + maxTime + " ms", activeOps.size() > 0);
        assertEquals("Multiple instances of " + opName + " present: " + activeOps, 1, activeOps.size());
        assertNotNull(opName + " not in status " + targetStatus + " after " + maxTime + " ms", opToCancel);

        return opToCancel;
    }

    private String findActiveOperation(DomainClient client, PathAddress address, String opName) throws Exception {
        ModelNode op = Util.createEmptyOperation(READ_CHILDREN_RESOURCES_OPERATION, address);
        op.get(CHILD_TYPE).set(ACTIVE_OPERATION);
        ModelNode result = executeForResult(op, client);
        if (result.isDefined()) {
            assertEquals(result.asString(), ModelType.OBJECT, result.getType());
            for (Property prop : result.asPropertyList()) {
                if (prop.getValue().get(OP).asString().equals(opName)) {
                    return prop.getName();
                }
            }
        }
        return null;
    }

    private void validateNoActiveOperation(DomainClient client, String host, String server) throws Exception {

        PathAddress baseAddress = getManagementControllerAddress(host, server);

        // The op should clear w/in a few ms but we'll wait up to 5 secs just in case
        // something strange is happening on the machine is overloaded
        long timeout = System.currentTimeMillis() + TimeoutUtil.adjust(5000);
        MgmtOperationException failure;
        do {
            String id = findActiveOperation(client, baseAddress, "block");
            if (id == null) {
                return;
            }
            failure = null;
            PathAddress address = baseAddress.append(PathElement.pathElement(ACTIVE_OPERATION, id));
            ModelNode op = Util.createEmptyOperation(READ_ATTRIBUTE_OPERATION, address);
            op.get(NAME).set(OP);
            try {
                executeForFailure(op, client);
            } catch (MgmtOperationException moe) {
                failure = moe;
            }
            Thread.sleep(50);
        } while (System.currentTimeMillis() < timeout);

        throw failure;
    }

    private void validateNoActiveOperation(DomainClient client, String host, String server, String id,
                                           boolean patient) throws Exception {
        PathAddress address = getManagementControllerAddress(host, server);
        address = address.append(PathElement.pathElement(ACTIVE_OPERATION, id));
        ModelNode op = Util.createEmptyOperation(READ_ATTRIBUTE_OPERATION, address);
        op.get(NAME).set(OP);

        // The op should clear w/in a few ms but we'll wait up to 5 secs just in case
        // something strange is happening on the machine is overloaded
        long timeout = patient ? System.currentTimeMillis() + TimeoutUtil.adjust(5000) : 0;
        MgmtOperationException failure;
        do {
            try {
                executeForFailure(op, client);
                return;
            } catch (MgmtOperationException moe) {
                if (!patient) {
                    throw moe;
                }
                failure = moe;
            }
            Thread.sleep(50);
        } while (System.currentTimeMillis() < timeout);

        throw failure;
    }

    private static PathAddress getManagementControllerAddress(String host, String server) {
        PathAddress address = PathAddress.pathAddress(PathElement.pathElement(HOST, host));
        if (server != null) {
            address = address.append(PathElement.pathElement(SERVER, server));
        }
        address = address.append(MGMT_CONTROLLER);
        return address;
    }

    private static ModelNode executeForResult(final ModelNode op, final ModelControllerClient modelControllerClient) throws IOException, MgmtOperationException {
        try {
            return DomainTestUtils.executeForResult(op, modelControllerClient);
        } catch (MgmtOperationException e) {
            System.out.println(" Op failed:");
            System.out.println(e.getOperation());
            System.out.println("with result");
            System.out.println(e.getResult());
            throw e;
        }
    }

    private static ModelNode executeForFailure(final ModelNode op, final ModelControllerClient modelControllerClient) throws IOException, MgmtOperationException {
        try {
            return DomainTestUtils.executeForFailure(op, modelControllerClient);
        } catch (MgmtOperationException e) {
            System.out.println(" Op that incorrectly succeeded:");
            System.out.println(e.getOperation());
            throw e;
        }
    }

    private static String getTestMethod() {
        final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
        for (StackTraceElement ste : stack) {
            String method = ste.getMethodName();
            if (method.startsWith("test")) {
                return method;
            }
        }
        return "unknown";
    }

    private static void assertFailedOrCancelled(ModelNode response) {
        assertXorCancelled(response, FAILED);
    }

    private static void assertSuccessOrCancelled(ModelNode response) {
        assertXorCancelled(response, SUCCESS);
    }

    private static void assertXorCancelled(ModelNode response, String x) {
        String outcome = response.get(OUTCOME).asString();
        if (!CANCELLED.equals(outcome)) {
            assertEquals(response.asString(), x, response.get(OUTCOME).asString());
        }

    }
}
