Order Book vs AMM for Derivatives: Trading Engine Design

The choice between order book and automated market maker (AMM) models fundamentally shapes a derivative exchange's characteristics. Order books offer precision and capital efficiency but require active market makers. AMMs provide passive liquidity but introduce slippage and impermanent loss. Understanding these trade-offs is crucial for building successful perpetual DEXs.

This comprehensive guide examines both models, their implementations, and innovative hybrid approaches.

Understanding the Trade-offs

Central Limit Order Book (CLOB) Design

Core Order Book Structure

contract CentralLimitOrderBook {
    struct Order {
        bytes32 id;
        address trader;
        bool isBuy;
        uint256 price;
        uint256 amount;
        uint256 filled;
        uint256 timestamp;
        bool active;
    }
    
    // Price -> Order IDs (sorted)
    mapping(uint256 => bytes32[]) public buyOrdersAtPrice;
    mapping(uint256 => bytes32[]) public sellOrdersAtPrice;
    
    // Sorted price levels
    uint256[] public buyPriceLevels;  // Descending
    uint256[] public sellPriceLevels; // Ascending
    
    mapping(bytes32 => Order) public orders;
    bytes32[] public allActiveOrders;
    
    uint256 public minOrderSize = 100e18; // $100
    uint256 public tickSize = 1e16;       // $0.01
    
    event OrderPlaced(
        bytes32 indexed orderId,
        address indexed trader,
        bool isBuy,
        uint256 price,
        uint256 amount
    );
    
    event OrderMatched(
        bytes32 indexed makerOrderId,
        bytes32 indexed takerOrderId,
        uint256 price,
        uint256 amount
    );
    
    function placeOrder(
        bool isBuy,
        uint256 price,
        uint256 amount
    ) external returns (bytes32 orderId) {
        require(amount >= minOrderSize, "Below minimum size");
        require(price % tickSize == 0, "Invalid tick size");
        
        // Lock collateral
        uint256 requiredCollateral = isBuy 
            ? (amount * price) / 1e18 
            : amount;
        collateralToken.transferFrom(msg.sender, address(this), requiredCollateral);
        
        orderId = keccak256(abi.encodePacked(
            msg.sender,
            block.timestamp,
            price,
            amount,
            nonce++
        ));
        
        orders[orderId] = Order({
            id: orderId,
            trader: msg.sender,
            isBuy: isBuy,
            price: price,
            amount: amount,
            filled: 0,
            timestamp: block.timestamp,
            active: true
        });
        
        // Add to price level
        if (isBuy) {
            buyOrdersAtPrice[price].push(orderId);
            _insertBuyPriceLevel(price);
        } else {
            sellOrdersAtPrice[price].push(orderId);
            _insertSellPriceLevel(price);
        }
        
        allActiveOrders.push(orderId);
        
        // Try to match immediately
        _matchOrder(orderId);
        
        emit OrderPlaced(orderId, msg.sender, isBuy, price, amount);
    }
    
    function _matchOrder(bytes32 orderId) internal {
        Order storage takerOrder = orders[orderId];
        if (!takerOrder.active || takerOrder.filled >= takerOrder.amount) return;
        
        uint256 remainingAmount = takerOrder.amount - takerOrder.filled;
        
        if (takerOrder.isBuy) {
            // Match against sell orders (lowest price first)
            for (uint i = 0; i < sellPriceLevels.length && remainingAmount > 0; i++) {
                uint256 sellPrice = sellPriceLevels[i];
                
                // Stop if sell price is above buy price
                if (sellPrice > takerOrder.price) break;
                
                bytes32[] storage sellOrders = sellOrdersAtPrice[sellPrice];
                
                for (uint j = 0; j < sellOrders.length && remainingAmount > 0; j++) {
                    bytes32 makerOrderId = sellOrders[j];
                    Order storage makerOrder = orders[makerOrderId];
                    
                    if (!makerOrder.active) continue;
                    
                    uint256 makerRemaining = makerOrder.amount - makerOrder.filled;
                    uint256 matchAmount = remainingAmount < makerRemaining 
                        ? remainingAmount 
                        : makerRemaining;
                    
                    // Execute match
                    _executeMatch(takerOrder, makerOrder, sellPrice, matchAmount);
                    
                    remainingAmount -= matchAmount;
                }
            }
        } else {
            // Match against buy orders (highest price first)
            for (uint i = 0; i < buyPriceLevels.length && remainingAmount > 0; i++) {
                uint256 buyPrice = buyPriceLevels[i];
                
                // Stop if buy price is below sell price
                if (buyPrice < takerOrder.price) break;
                
                bytes32[] storage buyOrders = buyOrdersAtPrice[buyPrice];
                
                for (uint j = 0; j < buyOrders.length && remainingAmount > 0; j++) {
                    bytes32 makerOrderId = buyOrders[j];
                    Order storage makerOrder = orders[makerOrderId];
                    
                    if (!makerOrder.active) continue;
                    
                    uint256 makerRemaining = makerOrder.amount - makerOrder.filled;
                    uint256 matchAmount = remainingAmount < makerRemaining 
                        ? remainingAmount 
                        : makerRemaining;
                    
                    _executeMatch(takerOrder, makerOrder, buyPrice, matchAmount);
                    
                    remainingAmount -= matchAmount;
                }
            }
        }
        
        // If fully filled, mark as inactive
        if (takerOrder.filled >= takerOrder.amount) {
            takerOrder.active = false;
        }
    }
    
    function _executeMatch(
        Order storage taker,
        Order storage maker,
        uint256 price,
        uint256 amount
    ) internal {
        taker.filled += amount;
        maker.filled += amount;
        
        if (maker.filled >= maker.amount) {
            maker.active = false;
        }
        
        // Update positions
        _updatePosition(taker.trader, taker.isBuy, amount, price);
        _updatePosition(maker.trader, maker.isBuy, amount, price);
        
        emit OrderMatched(maker.id, taker.id, price, amount);
    }
    
    function _insertBuyPriceLevel(uint256 price) internal {
        // Insert in descending order
        uint256 i = 0;
        while (i < buyPriceLevels.length && buyPriceLevels[i] > price) {
            i++;
        }
        
        if (i >= buyPriceLevels.length || buyPriceLevels[i] != price) {
            buyPriceLevels.push(0);
            for (uint j = buyPriceLevels.length - 1; j > i; j--) {
                buyPriceLevels[j] = buyPriceLevels[j-1];
            }
            buyPriceLevels[i] = price;
        }
    }
    
    function _insertSellPriceLevel(uint256 price) internal {
        // Insert in ascending order
        uint256 i = 0;
        while (i < sellPriceLevels.length && sellPriceLevels[i] < price) {
            i++;
        }
        
        if (i >= sellPriceLevels.length || sellPriceLevels[i] != price) {
            sellPriceLevels.push(0);
            for (uint j = sellPriceLevels.length - 1; j > i; j--) {
                sellPriceLevels[j] = sellPriceLevels[j-1];
            }
            sellPriceLevels[i] = price;
        }
    }
    
    function cancelOrder(bytes32 orderId) external {
        Order storage order = orders[orderId];
        require(order.trader == msg.sender, "Not order owner");
        require(order.active, "Order not active");
        
        order.active = false;
        
        // Return unfilled collateral
        uint256 unfilled = order.amount - order.filled;
        uint256 collateralToReturn = order.isBuy 
            ? (unfilled * order.price) / 1e18 
            : unfilled;
        
        collateralToken.transfer(msg.sender, collateralToReturn);
        
        emit OrderCancelled(orderId);
    }
    
    function getOrderBook(uint256 depth) external view returns (
        uint256[] memory bids,
        uint256[] memory bidSizes,
        uint256[] memory asks,
        uint256[] memory askSizes
    ) {
        bids = new uint256[](depth);
        bidSizes = new uint256[](depth);
        asks = new uint256[](depth);
        askSizes = new uint256[](depth);
        
        // Get top bids
        for (uint i = 0; i < depth && i < buyPriceLevels.length; i++) {
            uint256 price = buyPriceLevels[i];
            bids[i] = price;
            
            uint256 totalSize = 0;
            bytes32[] storage ordersAtPrice = buyOrdersAtPrice[price];
            for (uint j = 0; j < ordersAtPrice.length; j++) {
                Order storage order = orders[ordersAtPrice[j]];
                if (order.active) {
                    totalSize += (order.amount - order.filled);
                }
            }
            bidSizes[i] = totalSize;
        }
        
        // Get top asks
        for (uint i = 0; i < depth && i < sellPriceLevels.length; i++) {
            uint256 price = sellPriceLevels[i];
            asks[i] = price;
            
            uint256 totalSize = 0;
            bytes32[] storage ordersAtPrice = sellOrdersAtPrice[price];
            for (uint j = 0; j < ordersAtPrice.length; j++) {
                Order storage order = orders[ordersAtPrice[j]];
                if (order.active) {
                    totalSize += (order.amount - order.filled);
                }
            }
            askSizes[i] = totalSize;
        }
    }
}

Virtual Automated Market Maker (vAMM)

Virtual Pool Design

contract VirtualAMM {
    // Virtual reserves - no real tokens, just math
    uint256 public virtualBaseReserve;
    uint256 public virtualQuoteReserve;
    uint256 public constantK;
    
    // Real positions
    struct Position {
        address trader;
        int256 size; // Positive = long, negative = short
        uint256 entryPrice;
        uint256 collateral;
    }
    
    mapping(address => Position) public positions;
    int256 public totalLongSize;
    int256 public totalShortSize;
    
    constructor(uint256 _initialBase, uint256 _initialQuote) {
        virtualBaseReserve = _initialBase;
        virtualQuoteReserve = _initialQuote;
        constantK = _initialBase * _initialQuote;
    }
    
    function getMarkPrice() public view returns (uint256) {
        return (virtualQuoteReserve * 1e18) / virtualBaseReserve;
    }
    
    function openPosition(
        bool isLong,
        uint256 size,
        uint256 collateral
    ) external {
        uint256 entryPrice = getMarkPrice();
        
        if (isLong) {
            // Long: Buy base asset with quote
            // x * y = k
            // (x + dx) * (y - dy) = k
            // dy = y - k/(x+dx)
            
            uint256 newBaseReserve = virtualBaseReserve - size;
            uint256 newQuoteReserve = constantK / newBaseReserve;
            uint256 quotePaid = newQuoteReserve - virtualQuoteReserve;
            
            virtualBaseReserve = newBaseReserve;
            virtualQuoteReserve = newQuoteReserve;
            
            positions[msg.sender] = Position({
                trader: msg.sender,
                size: int256(size),
                entryPrice: entryPrice,
                collateral: collateral
            });
            
            totalLongSize += int256(size);
        } else {
            // Short: Sell base asset
            uint256 newBaseReserve = virtualBaseReserve + size;
            uint256 newQuoteReserve = constantK / newBaseReserve;
            uint256 quoteReceived = virtualQuoteReserve - newQuoteReserve;
            
            virtualBaseReserve = newBaseReserve;
            virtualQuoteReserve = newQuoteReserve;
            
            positions[msg.sender] = Position({
                trader: msg.sender,
                size: -int256(size),
                entryPrice: entryPrice,
                collateral: collateral
            });
            
            totalShortSize -= int256(size);
        }
        
        emit PositionOpened(msg.sender, isLong, size, entryPrice);
    }
    
    function closePosition() external {
        Position storage position = positions[msg.sender];
        require(position.size != 0, "No position");
        
        uint256 exitPrice = getMarkPrice();
        uint256 size = position.size > 0 ? uint256(position.size) : uint256(-position.size);
        
        if (position.size > 0) {
            // Close long
            uint256 newBaseReserve = virtualBaseReserve + size;
            uint256 newQuoteReserve = constantK / newBaseReserve;
            uint256 quoteReceived = virtualQuoteReserve - newQuoteReserve;
            
            virtualBaseReserve = newBaseReserve;
            virtualQuoteReserve = newQuoteReserve;
            
            totalLongSize -= position.size;
        } else {
            // Close short
            uint256 newBaseReserve = virtualBaseReserve - size;
            uint256 newQuoteReserve = constantK / newBaseReserve;
            uint256 quotePaid = newQuoteReserve - virtualQuoteReserve;
            
            virtualBaseReserve = newBaseReserve;
            virtualQuoteReserve = newQuoteReserve;
            
            totalShortSize += position.size;
        }
        
        delete positions[msg.sender];
        
        emit PositionClosed(msg.sender, exitPrice);
    }
    
    function calculateSlippage(uint256 size) external view returns (uint256) {
        uint256 priceBefore = getMarkPrice();
        
        uint256 newBaseReserve = virtualBaseReserve - size;
        uint256 newQuoteReserve = constantK / newBaseReserve;
        uint256 priceAfter = (newQuoteReserve * 1e18) / newBaseReserve;
        
        uint256 priceDiff = priceAfter > priceBefore 
            ? priceAfter - priceBefore 
            : priceBefore - priceAfter;
        
        return (priceDiff * 10000) / priceBefore; // In bps
    }
}

Hybrid Order Book-AMM

Combining Both Models

contract HybridTradingEngine {
    address public orderBook;
    address public amm;
    
    uint256 public constant AMM_THRESHOLD = 50000e18; // Use AMM for trades under $50k
    
    function executeTrade(
        bool isBuy,
        uint256 size,
        uint256 limitPrice,
        bool useLimitOrder
    ) external {
        if (useLimitOrder) {
            // Use order book for limit orders
            _placeLimitOrder(isBuy, limitPrice, size);
        } else {
            // Market order - choose best execution
            if (size < AMM_THRESHOLD) {
                // Small trades -> AMM (better for retail)
                _tradeOnAMM(isBuy, size);
            } else {
                // Large trades -> Order book (better price for size)
                _tradeOnOrderBook(isBuy, size);
            }
        }
    }
    
    function getBestPrice(bool isBuy, uint256 size) external view returns (uint256) {
        uint256 orderBookPrice = getOrderBookPrice(isBuy, size);
        uint256 ammPrice = getAMMPrice(isBuy, size);
        
        if (isBuy) {
            return orderBookPrice < ammPrice ? orderBookPrice : ammPrice;
        } else {
            return orderBookPrice > ammPrice ? orderBookPrice : ammPrice;
        }
    }
    
    function smartRoute(
        bool isBuy,
        uint256 size
    ) external {
        // Split large orders between AMM and order book
        uint256 ammSize = size / 4; // 25% to AMM
        uint256 orderBookSize = size - ammSize; // 75% to order book
        
        _tradeOnAMM(isBuy, ammSize);
        _tradeOnOrderBook(isBuy, orderBookSize);
    }
}

Concentrated Liquidity for Derivatives

Uniswap V3-Style for Perpetuals

contract ConcentratedLiquidityDerivatives {
    struct LiquidityPosition {
        int24 tickLower;
        int24 tickUpper;
        uint128 liquidity;
        uint256 feeGrowthInside0LastX128;
        uint256 feeGrowthInside1LastX128;
        address owner;
    }
    
    // Tick -> net liquidity change
    mapping(int24 => int128) public tickLiquidityNet;
    mapping(int24 => uint256) public tickFeesOutside0X128;
    mapping(int24 => uint256) public tickFeesOutside1X128;
    
    int24 public currentTick;
    uint256 public sqrtPriceX96;
    
    function addLiquidity(
        int24 tickLower,
        int24 tickUpper,
        uint128 liquidity
    ) external returns (uint256 positionId) {
        require(tickUpper > tickLower, "Invalid range");
        require(tickLower >= MIN_TICK && tickUpper <= MAX_TICK, "Out of bounds");
        
        // Calculate token amounts needed
        (uint256 amount0, uint256 amount1) = calculateLiquidityAmounts(
            tickLower,
            tickUpper,
            liquidity
        );
        
        // Update ticks
        tickLiquidityNet[tickLower] += int128(liquidity);
        tickLiquidityNet[tickUpper] -= int128(liquidity);
        
        positionId = positionCount++;
        positions[positionId] = LiquidityPosition({
            tickLower: tickLower,
            tickUpper: tickUpper,
            liquidity: liquidity,
            feeGrowthInside0LastX128: 0,
            feeGrowthInside1LastX128: 0,
            owner: msg.sender
        });
        
        // Transfer tokens
        if (amount0 > 0) token0.transferFrom(msg.sender, address(this), amount0);
        if (amount1 > 0) token1.transferFrom(msg.sender, address(this), amount1);
        
        emit LiquidityAdded(positionId, tickLower, tickUpper, liquidity);
    }
    
    function swap(
        bool zeroForOne,
        int256 amountSpecified,
        uint160 sqrtPriceLimitX96
    ) external returns (int256 amount0, int256 amount1) {
        // ... swap logic traversing ticks
        // Similar to Uniswap V3 but for derivatives
    }
    
    function calculateLiquidityAmounts(
        int24 tickLower,
        int24 tickUpper,
        uint128 liquidity
    ) internal view returns (uint256 amount0, uint256 amount1) {
        uint160 sqrtPriceLowerX96 = TickMath.getSqrtRatioAtTick(tickLower);
        uint160 sqrtPriceUpperX96 = TickMath.getSqrtRatioAtTick(tickUpper);
        
        if (currentTick < tickLower) {
            // Current price below range
            amount0 = SqrtPriceMath.getAmount0Delta(
                sqrtPriceLowerX96,
                sqrtPriceUpperX96,
                liquidity,
                true
            );
        } else if (currentTick < tickUpper) {
            // Current price within range
            amount0 = SqrtPriceMath.getAmount0Delta(
                sqrtPriceX96,
                sqrtPriceUpperX96,
                liquidity,
                true
            );
            amount1 = SqrtPriceMath.getAmount1Delta(
                sqrtPriceLowerX96,
                sqrtPriceX96,
                liquidity,
                true
            );
        } else {
            // Current price above range
            amount1 = SqrtPriceMath.getAmount1Delta(
                sqrtPriceLowerX96,
                sqrtPriceUpperX96,
                liquidity,
                true
            );
        }
    }
}

Liquidity Provision Incentives

Market Maker Rewards

contract MarketMakerIncentives {
    struct MMPerformance {
        address mm;
        uint256 volume24h;
        uint256 spreadImprovement;
        uint256 uptime;
        uint256 rewardScore;
    }
    
    mapping(address => MMPerformance) public mmPerformance;
    
    function calculateRewards(address mm) public view returns (uint256) {
        MMPerformance memory perf = mmPerformance[mm];
        
        // Volume component (40%)
        uint256 volumeScore = (perf.volume24h * 40) / TARGET_VOLUME;
        
        // Spread improvement (30%)
        uint256 spreadScore = (perf.spreadImprovement * 30) / 10000;
        
        // Uptime (30%)
        uint256 uptimeScore = (perf.uptime * 30) / 86400; // 24 hours
        
        uint256 totalScore = volumeScore + spreadScore + uptimeScore;
        
        // Rewards proportional to score
        return (totalRewardsPool * totalScore) / totalScores;
    }
    
    function distributeRewards() external {
        address[] memory marketMakers = getMarketMakers();
        
        for (uint i = 0; i < marketMakers.length; i++) {
            uint256 reward = calculateRewards(marketMakers[i]);
            rewardToken.transfer(marketMakers[i], reward);
        }
    }
}

MEV Protection

Batch Auctions

contract BatchAuctionDEX {
    struct BatchOrder {
        address trader;
        bool isBuy;
        uint256 minPrice;
        uint256 maxPrice;
        uint256 amount;
    }
    
    BatchOrder[] public currentBatch;
    uint256 public batchEndTime;
    uint256 public constant BATCH_DURATION = 5 minutes;
    
    function submitBatchOrder(
        bool isBuy,
        uint256 minPrice,
        uint256 maxPrice,
        uint256 amount
    ) external {
        require(block.timestamp < batchEndTime, "Batch closed");
        
        // Lock collateral
        collateralToken.transferFrom(msg.sender, address(this), amount);
        
        currentBatch.push(BatchOrder({
            trader: msg.sender,
            isBuy: isBuy,
            minPrice: minPrice,
            maxPrice: maxPrice,
            amount: amount
        }));
    }
    
    function settleBatch() external {
        require(block.timestamp >= batchEndTime, "Batch still open");
        
        // Sort orders by price
        BatchOrder[] memory sortedBuys = sortBuysByPrice(currentBatch);
        BatchOrder[] memory sortedSells = sortSellsByPrice(currentBatch);
        
        // Find clearing price (where demand = supply)
        uint256 clearingPrice = findClearingPrice(sortedBuys, sortedSells);
        
        // Execute all eligible orders at clearing price
        for (uint i = 0; i < currentBatch.length; i++) {
            BatchOrder memory order = currentBatch[i];
            
            if (order.isBuy && order.maxPrice >= clearingPrice) {
                _executeBuy(order.trader, clearingPrice, order.amount);
            } else if (!order.isBuy && order.minPrice <= clearingPrice) {
                _executeSell(order.trader, clearingPrice, order.amount);
            }
        }
        
        // Clear batch
        delete currentBatch;
        batchEndTime = block.timestamp + BATCH_DURATION;
    }
}

Best Practices

1. Gas Optimization

• Use off-chain order matching, on-chain settlement
• Batch multiple operations
• Optimize data structures

2. Liquidity Incentives

• Reward consistent market makers
• Provide rebates for maker orders
• Bootstrap with protocol liquidity

3. Fairness

• Time priority for orders at same price
• Prevent wash trading
• Transparent matching rules

4. Risk Management

• Position limits
• Circuit breakers
• Insurance funds

5. User Experience

• Fast order execution
• Low slippage for retail
• Deep liquidity for institutions

Conclusion

The choice between order book and AMM depends on target users:

Choose Order Book if:

• Targeting professional traders
• Need precise price control
• Can attract market makers
• Want capital efficiency

Choose AMM if:

• Targeting retail users
• Want passive liquidity
• Prioritize simplicity
• Accept some slippage

Choose Hybrid if:

• Serving diverse user base
• Want best of both worlds
• Can manage complexity
• Prioritize flexibility

The most successful perpetual DEXs will likely use hybrid models—order books for size and precision, AMMs for accessibility and passive liquidity. The key is matching the trading engine to your users' needs while maintaining security, fairness, and efficiency.