Change the interface to the type legalization method

ReplaceNodeResults: rather than returning a node which
must have the same number of results as the original
node (which means mucking around with MERGE_VALUES,
and which is also easy to get wrong since SelectionDAG
folding may mean you don't get the node you expect),
return the results in a vector.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@60348 91177308-0d34-0410-b5e6-96231b3b80d8

14 files changed, 335 insertions, 299 deletions

diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index c54ebdc..1f6294f 100644
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -1344,7 +1344,7 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG,
   return DAG.getNode(ISD::TokenFactor, MVT::Other, &TFOps[0], i);
 }
 
-static SDNode *ExpandBIT_CONVERT(SDNode *N, SelectionDAG &DAG) {
+static SDValue ExpandBIT_CONVERT(SDNode *N, SelectionDAG &DAG) {
   SDValue Op = N->getOperand(0);
   if (N->getValueType(0) == MVT::f64) {
     // Turn i64->f64 into FMDRR.
@@ -1352,7 +1352,7 @@ static SDNode *ExpandBIT_CONVERT(SDNode *N, SelectionDAG &DAG) {
                              DAG.getConstant(0, MVT::i32));
     SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
                              DAG.getConstant(1, MVT::i32));
-    return DAG.getNode(ARMISD::FMDRR, MVT::f64, Lo, Hi).getNode();
+    return DAG.getNode(ARMISD::FMDRR, MVT::f64, Lo, Hi);
   }
   
   // Turn f64->i64 into FMRRD.
@@ -1360,21 +1360,21 @@ static SDNode *ExpandBIT_CONVERT(SDNode *N, SelectionDAG &DAG) {
                             &Op, 1);
   
   // Merge the pieces into a single i64 value.
-  return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Cvt, Cvt.getValue(1)).getNode();
+  return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Cvt, Cvt.getValue(1));
 }
 
-static SDNode *ExpandSRx(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) {
+static SDValue ExpandSRx(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) {
   assert(N->getValueType(0) == MVT::i64 &&
          (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
          "Unknown shift to lower!");
-  
+
   // We only lower SRA, SRL of 1 here, all others use generic lowering.
   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
       cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 1)
-    return 0;
+    return SDValue();
   
   // If we are in thumb mode, we don't have RRX.
-  if (ST->isThumb()) return 0;
+  if (ST->isThumb()) return SDValue();
   
   // Okay, we have a 64-bit SRA or SRL of 1.  Lower this to an RRX expr.
   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(0),
@@ -1391,7 +1391,7 @@ static SDNode *ExpandSRx(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) {
   Lo = DAG.getNode(ARMISD::RRX, MVT::i32, Lo, Hi.getValue(1));
   
   // Merge the pieces into a single i64 value.
- return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi).getNode();
+ return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
 }
 
 
@@ -1419,22 +1419,34 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   case ISD::FRAMEADDR:     break;
   case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
-  case ISD::BIT_CONVERT:   return SDValue(ExpandBIT_CONVERT(Op.getNode(), DAG), 0);
+  case ISD::BIT_CONVERT:   return ExpandBIT_CONVERT(Op.getNode(), DAG);
   case ISD::SRL:
-  case ISD::SRA:           return SDValue(ExpandSRx(Op.getNode(), DAG,Subtarget),0);
+  case ISD::SRA:           return ExpandSRx(Op.getNode(), DAG,Subtarget);
   }
   return SDValue();
 }
 
 
-/// ReplaceNodeResults - Provide custom lowering hooks for nodes with illegal
-/// result types.
-SDNode *ARMTargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) {
+/// ReplaceNodeResults - Replace the results of node with an illegal result
+/// type with new values built out of custom code.
+///
+void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
+                                           SmallVectorImpl<SDValue>&Results,
+                                           SelectionDAG &DAG) {
   switch (N->getOpcode()) {
-  default: assert(0 && "Don't know how to custom expand this!"); abort();
-  case ISD::BIT_CONVERT:   return ExpandBIT_CONVERT(N, DAG);
+  default:
+    assert(0 && "Don't know how to custom expand this!");
+    return;
+  case ISD::BIT_CONVERT:
+    Results.push_back(ExpandBIT_CONVERT(N, DAG));
+    return;
   case ISD::SRL:
-  case ISD::SRA:           return ExpandSRx(N, DAG, Subtarget);
+  case ISD::SRA: {
+    SDValue Res = ExpandSRx(N, DAG, Subtarget);
+    if (Res.getNode())
+      Results.push_back(Res);
+    return;
+  }
   }
 }
   
diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index 8408cc5..3f664fc 100644
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -76,8 +76,13 @@ namespace llvm {
     explicit ARMTargetLowering(TargetMachine &TM);
 
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG);
-    virtual SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
-        
+
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
+    ///
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
+
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
     
     virtual const char *getTargetNodeName(unsigned Opcode) const;
diff --git a/lib/Target/Alpha/AlphaISelLowering.cpp b/lib/Target/Alpha/AlphaISelLowering.cpp
index 360db5f..66d66d0 100644
--- a/lib/Target/Alpha/AlphaISelLowering.cpp
+++ b/lib/Target/Alpha/AlphaISelLowering.cpp
@@ -612,15 +612,18 @@ SDValue AlphaTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   return SDValue();
 }
 
-SDNode *AlphaTargetLowering::ReplaceNodeResults(SDNode *N,
-                                                SelectionDAG &DAG) {
+void AlphaTargetLowering::ReplaceNodeResults(SDNode *N,
+                                             SmallVectorImpl<SDValue>&Results,
+                                             SelectionDAG &DAG) {
   assert(N->getValueType(0) == MVT::i32 &&
          N->getOpcode() == ISD::VAARG &&
          "Unknown node to custom promote!");
 
   SDValue Chain, DataPtr;
   LowerVAARG(N, Chain, DataPtr, DAG);
-  return DAG.getLoad(N->getValueType(0), Chain, DataPtr, NULL, 0).getNode();
+  SDValue Res = DAG.getLoad(N->getValueType(0), Chain, DataPtr, NULL, 0);
+  Results.push_back(Res);
+  Results.push_back(SDValue(Res.getNode(), 1));
 }
 
 
diff --git a/lib/Target/Alpha/AlphaISelLowering.h b/lib/Target/Alpha/AlphaISelLowering.h
index 3e870af..a29a518 100644
--- a/lib/Target/Alpha/AlphaISelLowering.h
+++ b/lib/Target/Alpha/AlphaISelLowering.h
@@ -72,7 +72,12 @@ namespace llvm {
     /// LowerOperation - Provide custom lowering hooks for some operations.
     ///
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG);
-    virtual SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
+
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
+    ///
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
 
     // Friendly names for dumps
     const char *getTargetNodeName(unsigned Opcode) const;
diff --git a/lib/Target/CellSPU/SPUISelLowering.cpp b/lib/Target/CellSPU/SPUISelLowering.cpp
index 6e64cae..bca8a41 100644
--- a/lib/Target/CellSPU/SPUISelLowering.cpp
+++ b/lib/Target/CellSPU/SPUISelLowering.cpp
@@ -2866,7 +2866,9 @@ SPUTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG)
   return SDValue();
 }
 
-SDNode *SPUTargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG)
+void SPUTargetLowering::ReplaceNodeResults(SDNode *N,
+                                           SmallVectorImpl<SDValue>&Results,
+                                           SelectionDAG &DAG)
 {
 #if 0
   unsigned Opc = (unsigned) N->getOpcode();
@@ -2885,7 +2887,6 @@ SDNode *SPUTargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG)
 #endif
 
   /* Otherwise, return unchanged */
-  return 0;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/CellSPU/SPUISelLowering.h b/lib/Target/CellSPU/SPUISelLowering.h
index a252ee3..fefaa68 100644
--- a/lib/Target/CellSPU/SPUISelLowering.h
+++ b/lib/Target/CellSPU/SPUISelLowering.h
@@ -113,9 +113,10 @@ namespace llvm {
     //! Custom lowering hooks
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG);
 
-    //! Provide custom lowering hooks for nodes with illegal result types.
-    SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
-    
+    //! Custom lowering hook for nodes with illegal result types.
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
+
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
     virtual void computeMaskedBitsForTargetNode(const SDValue Op,
diff --git a/lib/Target/PIC16/PIC16ISelLowering.cpp b/lib/Target/PIC16/PIC16ISelLowering.cpp
index 8a5fdb2..880fcc9 100644
--- a/lib/Target/PIC16/PIC16ISelLowering.cpp
+++ b/lib/Target/PIC16/PIC16ISelLowering.cpp
@@ -90,24 +90,35 @@ const char *PIC16TargetLowering::getTargetNodeName(unsigned Opcode) const {
   }
 }
 
-SDNode *PIC16TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) {
+void PIC16TargetLowering::ReplaceNodeResults(SDNode *N,
+                                             SmallVectorImpl<SDValue>&Results,
+                                             SelectionDAG &DAG) {
   switch (N->getOpcode()) {
     case ISD::GlobalAddress:
-      return ExpandGlobalAddress(N, DAG);
+      Results.push_back(ExpandGlobalAddress(N, DAG));
+      return;
     case ISD::STORE:
-      return ExpandStore(N, DAG);
+      Results.push_back(ExpandStore(N, DAG));
+      return;
     case ISD::LOAD:
-      return ExpandLoad(N, DAG);
+      Results.push_back(ExpandLoad(N, DAG));
+      return;
     case ISD::ADD:
-      return ExpandAdd(N, DAG);
-    case ISD::SHL:
-      return ExpandShift(N, DAG);
+//      return ExpandAdd(N, DAG);
+      return;
+    case ISD::SHL: {
+      SDValue Res = ExpandShift(N, DAG);
+      if (Res.getNode())
+        Results.push_back(Res);
+      return;
+    }
     default:
       assert (0 && "not implemented");
+      return;
   }
 }
 
-SDNode *PIC16TargetLowering::ExpandStore(SDNode *N, SelectionDAG &DAG) { 
+SDValue PIC16TargetLowering::ExpandStore(SDNode *N, SelectionDAG &DAG) {
   StoreSDNode *St = cast<StoreSDNode>(N);
   SDValue Chain = St->getChain();
   SDValue Src = St->getValue();
@@ -119,9 +130,8 @@ SDNode *PIC16TargetLowering::ExpandStore(SDNode *N, SelectionDAG &DAG) {
   LegalizeAddress(Ptr, DAG, PtrLo, PtrHi, StoreOffset);
  
   if (ValueType == MVT::i8) {
-    SDValue Store = DAG.getNode (PIC16ISD::PIC16Store, MVT::Other, Chain, Src,
-                                 PtrLo, PtrHi, DAG.getConstant (0, MVT::i8));
-    return Store.getNode();
+    return DAG.getNode (PIC16ISD::PIC16Store, MVT::Other, Chain, Src,
+                        PtrLo, PtrHi, DAG.getConstant (0, MVT::i8));
   }
   else if (ValueType == MVT::i16) {
     // Get the Lo and Hi parts from MERGE_VALUE or BUILD_PAIR.
@@ -142,7 +152,7 @@ SDNode *PIC16TargetLowering::ExpandStore(SDNode *N, SelectionDAG &DAG) {
                                  DAG.getConstant (1 + StoreOffset, MVT::i8));
 
     return DAG.getNode(ISD::TokenFactor, MVT::Other, getChain(Store1),
-                       getChain(Store2)).getNode();
+                       getChain(Store2));
   }
   else if (ValueType == MVT::i32) {
     // Get the Lo and Hi parts from MERGE_VALUE or BUILD_PAIR.
@@ -190,16 +200,16 @@ SDNode *PIC16TargetLowering::ExpandStore(SDNode *N, SelectionDAG &DAG) {
                                  getChain(Store2));
     SDValue RetHi =  DAG.getNode(ISD::TokenFactor, MVT::Other, getChain(Store3),
                                 getChain(Store4));
-    return  DAG.getNode(ISD::TokenFactor, MVT::Other, RetLo, RetHi).getNode();
-
+    return DAG.getNode(ISD::TokenFactor, MVT::Other, RetLo, RetHi);
   }
   else {
     assert (0 && "value type not supported");
+    return SDValue();
   }
 }
 
 // ExpandGlobalAddress - 
-SDNode *PIC16TargetLowering::ExpandGlobalAddress(SDNode *N, SelectionDAG &DAG) {
+SDValue PIC16TargetLowering::ExpandGlobalAddress(SDNode *N, SelectionDAG &DAG) {
   GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(SDValue(N, 0));
   
   SDValue TGA = DAG.getTargetGlobalAddress(G->getGlobal(), MVT::i8,
@@ -209,7 +219,7 @@ SDNode *PIC16TargetLowering::ExpandGlobalAddress(SDNode *N, SelectionDAG &DAG) {
   SDValue Hi = DAG.getNode(PIC16ISD::Hi, MVT::i8, TGA);
 
   SDValue BP = DAG.getNode(ISD::BUILD_PAIR, MVT::i16, Lo, Hi);
-  return BP.getNode();
+  return BP;
 }
 
 bool PIC16TargetLowering::isDirectAddress(const SDValue &Op) {
@@ -351,20 +361,20 @@ void PIC16TargetLowering:: LegalizeAddress(SDValue Ptr, SelectionDAG &DAG,
   return;
 }
 
-SDNode *PIC16TargetLowering::ExpandAdd(SDNode *N, SelectionDAG &DAG) {
-  SDValue OperLeft = N->getOperand(0);
-  SDValue OperRight = N->getOperand(1);
-
-  if((OperLeft.getOpcode() == ISD::Constant) || 
-     (OperRight.getOpcode() == ISD::Constant)) { 
-    return NULL;
-  }
-
-  // These case are yet to be handled
-  return NULL;
-}
+//SDNode *PIC16TargetLowering::ExpandAdd(SDNode *N, SelectionDAG &DAG) {
+//  SDValue OperLeft = N->getOperand(0);
+//  SDValue OperRight = N->getOperand(1);
+//
+//  if((OperLeft.getOpcode() == ISD::Constant) ||
+//     (OperRight.getOpcode() == ISD::Constant)) {
+//    return NULL;
+//  }
+//
+//  // These case are yet to be handled
+//  return NULL;
+//}
 
-SDNode *PIC16TargetLowering::ExpandLoad(SDNode *N, SelectionDAG &DAG) {
+SDValue PIC16TargetLowering::ExpandLoad(SDNode *N, SelectionDAG &DAG) {
   LoadSDNode *LD = dyn_cast<LoadSDNode>(SDValue(N, 0));
   SDValue Chain = LD->getChain();
   SDValue Ptr = LD->getBasePtr();
@@ -438,7 +448,7 @@ SDNode *PIC16TargetLowering::ExpandLoad(SDNode *N, SelectionDAG &DAG) {
 
   if (VT == MVT::i8) {
     // Operand of Load is illegal -- Load itself is legal
-    return PICLoads[0].getNode();
+    return PICLoads[0];
   }
   else if (VT == MVT::i16) {
     BP = DAG.getNode(ISD::BUILD_PAIR, VT, PICLoads[0], PICLoads[1]);
@@ -467,12 +477,10 @@ SDNode *PIC16TargetLowering::ExpandLoad(SDNode *N, SelectionDAG &DAG) {
     }
   }
   Tys = DAG.getVTList(VT, MVT::Other); 
-  SDValue MergeV = DAG.getNode(ISD::MERGE_VALUES, Tys, BP, Chain);
-  return MergeV.getNode();
-
+  return DAG.getNode(ISD::MERGE_VALUES, Tys, BP, Chain);
 }
 
-SDNode *PIC16TargetLowering::ExpandShift(SDNode *N, SelectionDAG &DAG) {
+SDValue PIC16TargetLowering::ExpandShift(SDNode *N, SelectionDAG &DAG) {
   SDValue Value = N->getOperand(0);
   SDValue Amt = N->getOperand(1);
   SDValue BCF, BCFInput;
@@ -483,11 +491,11 @@ SDNode *PIC16TargetLowering::ExpandShift(SDNode *N, SelectionDAG &DAG) {
   
   // Currently handling Constant shift only
   if (Amt.getOpcode() != ISD::Constant)
-    return NULL;
+    return SDValue();
 
   // Following code considers 16 bit left-shift only
   if (N->getValueType(0) != MVT::i16)
-    return NULL;
+    return SDValue();
 
   if (N->getOpcode() == ISD::SHL) {
     ShfNode = PIC16ISD::LSLF;
@@ -515,8 +523,7 @@ SDNode *PIC16TargetLowering::ExpandShift(SDNode *N, SelectionDAG &DAG) {
     BCFInput = RotCom.getValue(1); 
   }
 
-  SDValue BP = DAG.getNode(ISD::BUILD_PAIR, N->getValueType(0), ShfCom, RotCom);
-  return BP.getNode();
+  return DAG.getNode(ISD::BUILD_PAIR, N->getValueType(0), ShfCom, RotCom);
 }
 
 SDValue PIC16TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
@@ -532,11 +539,11 @@ SDValue PIC16TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
     case ISD::SUBC:
       return LowerSUBC(Op, DAG);
     case ISD::LOAD:
-      return SDValue(ExpandLoad(Op.getNode(), DAG), Op.getResNo());
+      return ExpandLoad(Op.getNode(), DAG);
     case ISD::STORE:
-      return SDValue(ExpandStore(Op.getNode(), DAG), Op.getResNo());
+      return ExpandStore(Op.getNode(), DAG);
     case ISD::SHL:
-      return SDValue(ExpandShift(Op.getNode(), DAG), Op.getResNo());
+      return ExpandShift(Op.getNode(), DAG);
     case ISD::OR:
     case ISD::AND:
     case ISD::XOR:
diff --git a/lib/Target/PIC16/PIC16ISelLowering.h b/lib/Target/PIC16/PIC16ISelLowering.h
index a2af24c..de8bd2b 100644
--- a/lib/Target/PIC16/PIC16ISelLowering.h
+++ b/lib/Target/PIC16/PIC16ISelLowering.h
@@ -66,12 +66,17 @@ namespace llvm {
     SDValue LowerSUBC(SDValue Op, SelectionDAG &DAG);
     SDValue LowerBinOp(SDValue Op, SelectionDAG &DAG);
 
-    SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandStore(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandLoad(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandAdd(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandGlobalAddress(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandShift(SDNode *N, SelectionDAG &DAG);
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
+    ///
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
+
+    SDValue ExpandStore(SDNode *N, SelectionDAG &DAG);
+    SDValue ExpandLoad(SDNode *N, SelectionDAG &DAG);
+//    SDNode *ExpandAdd(SDNode *N, SelectionDAG &DAG);
+    SDValue ExpandGlobalAddress(SDNode *N, SelectionDAG &DAG);
+    SDValue ExpandShift(SDNode *N, SelectionDAG &DAG);
 
     SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; 
     SDValue PerformPIC16LoadCombine(SDNode *N, DAGCombinerInfo &DCI) const; 
diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp
index b02acaf..d72fdf6 100644
--- a/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -2874,66 +2874,6 @@ SDValue PPCTargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) {
   return DAG.getLoad(Op.getValueType(), Chain, FIPtr, NULL, 0);
 }
 
-SDValue PPCTargetLowering::LowerFP_ROUND_INREG(SDValue Op, 
-                                                 SelectionDAG &DAG) {
-  assert(Op.getValueType() == MVT::ppcf128);
-  SDNode *Node = Op.getNode();
-  assert(Node->getOperand(0).getValueType() == MVT::ppcf128);
-  SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::f64, Node->getOperand(0),
-                           DAG.getIntPtrConstant(0));
-  SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::f64, Node->getOperand(0),
-                           DAG.getIntPtrConstant(1));
-
-  // This sequence changes FPSCR to do round-to-zero, adds the two halves
-  // of the long double, and puts FPSCR back the way it was.  We do not
-  // actually model FPSCR.
-  std::vector<MVT> NodeTys;
-  SDValue Ops[4], Result, MFFSreg, InFlag, FPreg;
-
-  NodeTys.push_back(MVT::f64);   // Return register
-  NodeTys.push_back(MVT::Flag);    // Returns a flag for later insns
-  Result = DAG.getNode(PPCISD::MFFS, NodeTys, &InFlag, 0);
-  MFFSreg = Result.getValue(0);
-  InFlag = Result.getValue(1);
-
-  NodeTys.clear();
-  NodeTys.push_back(MVT::Flag);   // Returns a flag
-  Ops[0] = DAG.getConstant(31, MVT::i32);
-  Ops[1] = InFlag;
-  Result = DAG.getNode(PPCISD::MTFSB1, NodeTys, Ops, 2);
-  InFlag = Result.getValue(0);
-
-  NodeTys.clear();
-  NodeTys.push_back(MVT::Flag);   // Returns a flag
-  Ops[0] = DAG.getConstant(30, MVT::i32);
-  Ops[1] = InFlag;
-  Result = DAG.getNode(PPCISD::MTFSB0, NodeTys, Ops, 2);
-  InFlag = Result.getValue(0);
-
-  NodeTys.clear();
-  NodeTys.push_back(MVT::f64);    // result of add
-  NodeTys.push_back(MVT::Flag);   // Returns a flag
-  Ops[0] = Lo;
-  Ops[1] = Hi;
-  Ops[2] = InFlag;
-  Result = DAG.getNode(PPCISD::FADDRTZ, NodeTys, Ops, 3);
-  FPreg = Result.getValue(0);
-  InFlag = Result.getValue(1);
-
-  NodeTys.clear();
-  NodeTys.push_back(MVT::f64);
-  Ops[0] = DAG.getConstant(1, MVT::i32);
-  Ops[1] = MFFSreg;
-  Ops[2] = FPreg;
-  Ops[3] = InFlag;
-  Result = DAG.getNode(PPCISD::MTFSF, NodeTys, Ops, 4);
-  FPreg = Result.getValue(0);
-
-  // We know the low half is about to be thrown away, so just use something
-  // convenient.
-  return DAG.getNode(ISD::BUILD_PAIR, MVT::ppcf128, FPreg, FPreg);
-}
-
 SDValue PPCTargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
   // Don't handle ppc_fp128 here; let it be lowered to a libcall.
   if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
@@ -3874,7 +3814,6 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   case ISD::SELECT_CC:          return LowerSELECT_CC(Op, DAG);
   case ISD::FP_TO_SINT:         return LowerFP_TO_SINT(Op, DAG);
   case ISD::SINT_TO_FP:         return LowerSINT_TO_FP(Op, DAG);
-  case ISD::FP_ROUND_INREG:     return LowerFP_ROUND_INREG(Op, DAG);
   case ISD::FLT_ROUNDS_:        return LowerFLT_ROUNDS_(Op, DAG);
 
   // Lower 64-bit shifts.
@@ -3896,17 +3835,74 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   return SDValue();
 }
 
-SDNode *PPCTargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) {
+void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
+                                           SmallVectorImpl<SDValue>&Results,
+                                           SelectionDAG &DAG) {
   switch (N->getOpcode()) {
   default:
-    return PPCTargetLowering::LowerOperation(SDValue (N, 0), DAG).getNode();
-  case ISD::FP_TO_SINT: {
-    SDValue Res = LowerFP_TO_SINT(SDValue(N, 0), DAG);
-    // Use MERGE_VALUES to drop the chain result value and get a node with one
-    // result.  This requires turning off getMergeValues simplification, since
-    // otherwise it will give us Res back.
-    return DAG.getMergeValues(&Res, 1, false).getNode();
+    assert(false && "Do not know how to custom type legalize this operation!");
+    return;
+  case ISD::FP_ROUND_INREG: {
+    assert(N->getValueType(0) == MVT::ppcf128);
+    assert(N->getOperand(0).getValueType() == MVT::ppcf128);
+    SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::f64, N->getOperand(0),
+                             DAG.getIntPtrConstant(0));
+    SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::f64, N->getOperand(0),
+                             DAG.getIntPtrConstant(1));
+
+    // This sequence changes FPSCR to do round-to-zero, adds the two halves
+    // of the long double, and puts FPSCR back the way it was.  We do not
+    // actually model FPSCR.
+    std::vector<MVT> NodeTys;
+    SDValue Ops[4], Result, MFFSreg, InFlag, FPreg;
+
+    NodeTys.push_back(MVT::f64);   // Return register
+    NodeTys.push_back(MVT::Flag);    // Returns a flag for later insns
+    Result = DAG.getNode(PPCISD::MFFS, NodeTys, &InFlag, 0);
+    MFFSreg = Result.getValue(0);
+    InFlag = Result.getValue(1);
+
+    NodeTys.clear();
+    NodeTys.push_back(MVT::Flag);   // Returns a flag
+    Ops[0] = DAG.getConstant(31, MVT::i32);
+    Ops[1] = InFlag;
+    Result = DAG.getNode(PPCISD::MTFSB1, NodeTys, Ops, 2);
+    InFlag = Result.getValue(0);
+
+    NodeTys.clear();
+    NodeTys.push_back(MVT::Flag);   // Returns a flag
+    Ops[0] = DAG.getConstant(30, MVT::i32);
+    Ops[1] = InFlag;
+    Result = DAG.getNode(PPCISD::MTFSB0, NodeTys, Ops, 2);
+    InFlag = Result.getValue(0);
+
+    NodeTys.clear();
+    NodeTys.push_back(MVT::f64);    // result of add
+    NodeTys.push_back(MVT::Flag);   // Returns a flag
+    Ops[0] = Lo;
+    Ops[1] = Hi;
+    Ops[2] = InFlag;
+    Result = DAG.getNode(PPCISD::FADDRTZ, NodeTys, Ops, 3);
+    FPreg = Result.getValue(0);
+    InFlag = Result.getValue(1);
+
+    NodeTys.clear();
+    NodeTys.push_back(MVT::f64);
+    Ops[0] = DAG.getConstant(1, MVT::i32);
+    Ops[1] = MFFSreg;
+    Ops[2] = FPreg;
+    Ops[3] = InFlag;
+    Result = DAG.getNode(PPCISD::MTFSF, NodeTys, Ops, 4);
+    FPreg = Result.getValue(0);
+
+    // We know the low half is about to be thrown away, so just use something
+    // convenient.
+    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::ppcf128, FPreg, FPreg));
+    return;
   }
+  case ISD::FP_TO_SINT:
+    Results.push_back(LowerFP_TO_SINT(SDValue(N, 0), DAG));
+    return;
   }
 }
 
diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h
index 00f707d..8fa7972 100644
--- a/lib/Target/PowerPC/PPCISelLowering.h
+++ b/lib/Target/PowerPC/PPCISelLowering.h
@@ -269,8 +269,12 @@ namespace llvm {
     ///
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG);
 
-    virtual SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
-    
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
+    ///
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
+
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
     
     virtual void computeMaskedBitsForTargetNode(const SDValue Op,
@@ -372,7 +376,6 @@ namespace llvm {
     SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG);
     SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG);
     SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG);
-    SDValue LowerFP_ROUND_INREG(SDValue Op, SelectionDAG &DAG);
     SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG);
     SDValue LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG);
     SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG);
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index eae1b11..0540538 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -4863,22 +4863,6 @@ SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) {
   return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0);
 }
 
-SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) {
-  std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(SDValue(N, 0), DAG);
-  SDValue FIST = Vals.first, StackSlot = Vals.second;
-  if (FIST.getNode() == 0) return 0;
-
-  MVT VT = N->getValueType(0);
-
-  // Return a load from the stack slot.
-  SDValue Res = DAG.getLoad(VT, FIST, StackSlot, NULL, 0);
-
-  // Use MERGE_VALUES to drop the chain result value and get a node with one
-  // result.  This requires turning off getMergeValues simplification, since
-  // otherwise it will give us Res back.
-  return DAG.getMergeValues(&Res, 1, false).getNode();
-}
-
 SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) {
   MVT VT = Op.getValueType();
   MVT EltVT = VT;
@@ -5550,36 +5534,6 @@ X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG,
   return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size());
 }
 
-/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain
-SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){
-  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
-  SDValue TheChain = N->getOperand(0);
-  SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1);
-  if (Subtarget->is64Bit()) {
-    SDValue rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1));
-    SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX,
-                                       MVT::i64, rax.getValue(2));
-    SDValue Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx,
-                                DAG.getConstant(32, MVT::i8));
-    SDValue Ops[] = {
-      DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1)
-    };
-    
-    return DAG.getMergeValues(Ops, 2).getNode();
-  }
-  
-  SDValue eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1));
-  SDValue edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX,
-                                       MVT::i32, eax.getValue(2));
-  // Use a buildpair to merge the two 32-bit values into a 64-bit one. 
-  SDValue Ops[] = { eax, edx };
-  Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2);
-
-  // Use a MERGE_VALUES to return the value and chain.
-  Ops[1] = edx.getValue(1);
-  return DAG.getMergeValues(Ops, 2).getNode();
-}
-
 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) {
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
 
@@ -6186,10 +6140,8 @@ SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) {
   case MVT::i16: Reg = X86::AX;  size = 2; break;
   case MVT::i32: Reg = X86::EAX; size = 4; break;
   case MVT::i64: 
-    if (Subtarget->is64Bit()) {
-      Reg = X86::RAX; size = 8;
-    } else //Should go away when LegalizeType stuff lands
-      return SDValue(ExpandATOMIC_CMP_SWAP(Op.getNode(), DAG), 0);
+    assert(Subtarget->is64Bit() && "Node not type legal!");
+    Reg = X86::RAX; size = 8;
     break;
   };
   SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg,
@@ -6206,66 +6158,22 @@ SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) {
   return cpOut;
 }
 
-SDNode* X86TargetLowering::ExpandATOMIC_CMP_SWAP(SDNode* Op,
+SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
                                                  SelectionDAG &DAG) {
-  MVT T = Op->getValueType(0);
-  assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
-  SDValue cpInL, cpInH;
-  cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2),
-                      DAG.getConstant(0, MVT::i32));
-  cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(2),
-                      DAG.getConstant(1, MVT::i32));
-  cpInL = DAG.getCopyToReg(Op->getOperand(0), X86::EAX,
-                           cpInL, SDValue());
-  cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX,
-                           cpInH, cpInL.getValue(1));
-  SDValue swapInL, swapInH;
-  swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3),
-                        DAG.getConstant(0, MVT::i32));
-  swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op->getOperand(3),
-                        DAG.getConstant(1, MVT::i32));
-  swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX,
-                             swapInL, cpInH.getValue(1));
-  swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX,
-                             swapInH, swapInL.getValue(1));
-  SDValue Ops[] = { swapInH.getValue(0),
-                    Op->getOperand(1),
-                    swapInH.getValue(1) };
+  assert(Subtarget->is64Bit() && "Result not type legalized?");
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
-  SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3);
-  SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, 
-                                        Result.getValue(1));
-  SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, 
-                                        cpOutL.getValue(2));
-  SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
-  SDValue ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2);
-  SDValue Vals[2] = { ResultVal, cpOutH.getValue(1) };
-  return DAG.getMergeValues(Vals, 2).getNode();
-}
-
-SDValue X86TargetLowering::LowerATOMIC_BINARY_64(SDValue Op,
-                                                 SelectionDAG &DAG,
-                                                 unsigned NewOp) {
-  SDNode *Node = Op.getNode();
-  MVT T = Node->getValueType(0);
-  assert (T == MVT::i64 && "Only know how to expand i64 atomics");
-  
-  SDValue Chain = Node->getOperand(0);
-  SDValue In1 = Node->getOperand(1);
-  SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
-                             Node->getOperand(2), DAG.getIntPtrConstant(0));
-  SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
-                             Node->getOperand(2), DAG.getIntPtrConstant(1));
-  // This is a generalized SDNode, not an AtomicSDNode, so it doesn't
-  // have a MemOperand.  Pass the info through as a normal operand.
-  SDValue LSI = DAG.getMemOperand(cast<MemSDNode>(Node)->getMemOperand());
-  SDValue Ops[] = { Chain, In1, In2L, In2H, LSI };
-  SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
-  SDValue Result = DAG.getNode(NewOp, Tys, Ops, 5);
-  SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
-  SDValue ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2);
-  SDValue Vals[2] = { ResultVal, Result.getValue(2) };
-  return SDValue(DAG.getMergeValues(Vals, 2).getNode(), 0);
+  SDValue TheChain = Op.getOperand(0);
+  SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1);
+  SDValue rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1));
+  SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, MVT::i64,
+                                   rax.getValue(2));
+  SDValue Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx,
+                            DAG.getConstant(32, MVT::i8));
+  SDValue Ops[] = {
+    DAG.getNode(ISD::OR, MVT::i64, rax, Tmp),
+    rdx.getValue(1)
+  };
+  return DAG.getMergeValues(Ops, 2);
 }
 
 SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
@@ -6298,22 +6206,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   case ISD::ATOMIC_LOAD_SUB_8:
   case ISD::ATOMIC_LOAD_SUB_16:
   case ISD::ATOMIC_LOAD_SUB_32: return LowerLOAD_SUB(Op,DAG);
-  case ISD::ATOMIC_LOAD_SUB_64: return (Subtarget->is64Bit()) ?
-                                        LowerLOAD_SUB(Op,DAG) :
-                                        LowerATOMIC_BINARY_64(Op,DAG,
-                                        X86ISD::ATOMSUB64_DAG);
-  case ISD::ATOMIC_LOAD_AND_64: return LowerATOMIC_BINARY_64(Op,DAG,
-                                        X86ISD::ATOMAND64_DAG);
-  case ISD::ATOMIC_LOAD_OR_64:  return LowerATOMIC_BINARY_64(Op, DAG,
-                                        X86ISD::ATOMOR64_DAG);
-  case ISD::ATOMIC_LOAD_XOR_64: return LowerATOMIC_BINARY_64(Op,DAG,
-                                        X86ISD::ATOMXOR64_DAG);
-  case ISD::ATOMIC_LOAD_NAND_64:return LowerATOMIC_BINARY_64(Op,DAG,
-                                        X86ISD::ATOMNAND64_DAG);
-  case ISD::ATOMIC_LOAD_ADD_64: return LowerATOMIC_BINARY_64(Op,DAG,
-                                        X86ISD::ATOMADD64_DAG);
-  case ISD::ATOMIC_SWAP_64:     return LowerATOMIC_BINARY_64(Op,DAG,
-                                        X86ISD::ATOMSWAP64_DAG);
+  case ISD::ATOMIC_LOAD_SUB_64: return LowerLOAD_SUB(Op,DAG);
   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
@@ -6356,22 +6249,120 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   case ISD::CTTZ:               return LowerCTTZ(Op, DAG);
   case ISD::SADDO:              return LowerXADDO(Op, DAG, ISD::SADDO);
   case ISD::UADDO:              return LowerXADDO(Op, DAG, ISD::UADDO);
-      
-  // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands.
-  case ISD::READCYCLECOUNTER:
-    return SDValue(ExpandREADCYCLECOUNTER(Op.getNode(), DAG), 0);
+  case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, DAG);
   }
 }
 
+void X86TargetLowering::
+ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results,
+                        SelectionDAG &DAG, unsigned NewOp) {
+  MVT T = Node->getValueType(0);
+  assert (T == MVT::i64 && "Only know how to expand i64 atomics");
+
+  SDValue Chain = Node->getOperand(0);
+  SDValue In1 = Node->getOperand(1);
+  SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
+                             Node->getOperand(2), DAG.getIntPtrConstant(0));
+  SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
+                             Node->getOperand(2), DAG.getIntPtrConstant(1));
+  // This is a generalized SDNode, not an AtomicSDNode, so it doesn't
+  // have a MemOperand.  Pass the info through as a normal operand.
+  SDValue LSI = DAG.getMemOperand(cast<MemSDNode>(Node)->getMemOperand());
+  SDValue Ops[] = { Chain, In1, In2L, In2H, LSI };
+  SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
+  SDValue Result = DAG.getNode(NewOp, Tys, Ops, 5);
+  SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
+  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2));
+  Results.push_back(Result.getValue(2));
+}
+
 /// ReplaceNodeResults - Replace a node with an illegal result type
 /// with a new node built out of custom code.
-SDNode *X86TargetLowering::ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) {
+void X86TargetLowering::ReplaceNodeResults(SDNode *N,
+                                           SmallVectorImpl<SDValue>&Results,
+                                           SelectionDAG &DAG) {
   switch (N->getOpcode()) {
   default:
-    return X86TargetLowering::LowerOperation(SDValue (N, 0), DAG).getNode();
-  case ISD::FP_TO_SINT:         return ExpandFP_TO_SINT(N, DAG);
-  case ISD::READCYCLECOUNTER:   return ExpandREADCYCLECOUNTER(N, DAG);
-  case ISD::ATOMIC_CMP_SWAP_64: return ExpandATOMIC_CMP_SWAP(N, DAG);
+    assert(false && "Do not know how to custom type legalize this operation!");
+    return;
+  case ISD::FP_TO_SINT: {
+    std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(SDValue(N, 0), DAG);
+    SDValue FIST = Vals.first, StackSlot = Vals.second;
+    if (FIST.getNode() != 0) {
+      MVT VT = N->getValueType(0);
+      // Return a load from the stack slot.
+      Results.push_back(DAG.getLoad(VT, FIST, StackSlot, NULL, 0));
+    }
+    return;
+  }
+  case ISD::READCYCLECOUNTER: {
+    SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
+    SDValue TheChain = N->getOperand(0);
+    SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1);
+    SDValue eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1));
+    SDValue edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, MVT::i32,
+                                     eax.getValue(2));
+    // Use a buildpair to merge the two 32-bit values into a 64-bit one.
+    SDValue Ops[] = { eax, edx };
+    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2));
+    Results.push_back(edx.getValue(1));
+    return;
+  }
+  case ISD::ATOMIC_CMP_SWAP_64: {
+    MVT T = N->getValueType(0);
+    assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
+    SDValue cpInL, cpInH;
+    cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(2),
+                        DAG.getConstant(0, MVT::i32));
+    cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(2),
+                        DAG.getConstant(1, MVT::i32));
+    cpInL = DAG.getCopyToReg(N->getOperand(0), X86::EAX, cpInL, SDValue());
+    cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, cpInH,
+                             cpInL.getValue(1));
+    SDValue swapInL, swapInH;
+    swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(3),
+                          DAG.getConstant(0, MVT::i32));
+    swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(3),
+                          DAG.getConstant(1, MVT::i32));
+    swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, swapInL,
+                               cpInH.getValue(1));
+    swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, swapInH,
+                               swapInL.getValue(1));
+    SDValue Ops[] = { swapInH.getValue(0),
+                      N->getOperand(1),
+                      swapInH.getValue(1) };
+    SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
+    SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3);
+    SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32,
+                                        Result.getValue(1));
+    SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32,
+                                        cpOutL.getValue(2));
+    SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
+    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2));
+    Results.push_back(cpOutH.getValue(1));
+    return;
+  }
+  case ISD::ATOMIC_LOAD_ADD_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_AND_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_NAND_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_OR_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_SUB_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG);
+    return;
+  case ISD::ATOMIC_LOAD_XOR_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG);
+    return;
+  case ISD::ATOMIC_SWAP_64:
+    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG);
+    return;
   }
 }
 
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index e9d8e55..062ab2e 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -375,10 +375,11 @@ namespace llvm {
     ///
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG);
 
-    /// ReplaceNodeResults - Replace a node with an illegal result type
-    /// with a new node built out of custom code.
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
     ///
-    virtual SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
 
     
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
@@ -596,12 +597,11 @@ namespace llvm {
 
     SDValue LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG);
     SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG);
-    SDValue LowerATOMIC_BINARY_64(SDValue Op, SelectionDAG &DAG, 
-                                  unsigned NewOp);
-    SDNode *ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG);
-    SDNode *ExpandATOMIC_CMP_SWAP(SDNode *N, SelectionDAG &DAG);
-    
+    SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG);
+
+    void ReplaceATOMIC_BINARY_64(SDNode *N, SmallVectorImpl<SDValue> &Results,
+                                 SelectionDAG &DAG, unsigned NewOp);
+
     SDValue EmitTargetCodeForMemset(SelectionDAG &DAG,
                                     SDValue Chain,
                                     SDValue Dst, SDValue Src,
diff --git a/lib/Target/XCore/XCoreISelLowering.cpp b/lib/Target/XCore/XCoreISelLowering.cpp
index 4e06c24..e24b998 100644
--- a/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/lib/Target/XCore/XCoreISelLowering.cpp
@@ -161,7 +161,7 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) {
   case ISD::VASTART:          return LowerVASTART(Op, DAG);
   // FIXME: Remove these when LegalizeDAGTypes lands.
   case ISD::ADD:
-  case ISD::SUB:              return SDValue(ExpandADDSUB(Op.getNode(), DAG),0);
+  case ISD::SUB:              return ExpandADDSUB(Op.getNode(), DAG);
   case ISD::FRAMEADDR:        return LowerFRAMEADDR(Op, DAG);
   default:
     assert(0 && "unimplemented operand");
@@ -169,16 +169,19 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) {
   }
 }
 
-/// ReplaceNodeResults - Provide custom lowering hooks for nodes with illegal
-/// result types.
-SDNode *XCoreTargetLowering::
-ReplaceNodeResults(SDNode *N, SelectionDAG &DAG) {
+/// ReplaceNodeResults - Replace the results of node with an illegal result
+/// type with new values built out of custom code.
+void XCoreTargetLowering::ReplaceNodeResults(SDNode *N,
+                                             SmallVectorImpl<SDValue>&Results,
+                                             SelectionDAG &DAG) {
   switch (N->getOpcode()) {
   default:
     assert(0 && "Don't know how to custom expand this!");
-    return NULL;
+    return;
   case ISD::ADD:
-  case ISD::SUB: return ExpandADDSUB(N, DAG);
+  case ISD::SUB:
+    Results.push_back(ExpandADDSUB(N, DAG));
+    return;
   }
 }
 
@@ -296,7 +299,7 @@ LowerJumpTable(SDValue Op, SelectionDAG &DAG)
   return DAG.getNode(XCoreISD::DPRelativeWrapper, MVT::i32, JTI);
 }
 
-SDNode *XCoreTargetLowering::
+SDValue XCoreTargetLowering::
 ExpandADDSUB(SDNode *N, SelectionDAG &DAG)
 {
   assert(N->getValueType(0) == MVT::i64 &&
@@ -326,7 +329,7 @@ ExpandADDSUB(SDNode *N, SelectionDAG &DAG)
                                   LHSH, RHSH, Carry);
   SDValue Hi(Ignored.getNode(), 1);
   // Merge the pieces
-  return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi).getNode();
+  return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
 }
 
 SDValue XCoreTargetLowering::
diff --git a/lib/Target/XCore/XCoreISelLowering.h b/lib/Target/XCore/XCoreISelLowering.h
index 0ce1c76..5417d7b 100644
--- a/lib/Target/XCore/XCoreISelLowering.h
+++ b/lib/Target/XCore/XCoreISelLowering.h
@@ -68,7 +68,11 @@ namespace llvm {
     /// LowerOperation - Provide custom lowering hooks for some operations.
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG);
 
-    virtual SDNode *ReplaceNodeResults(SDNode *N, SelectionDAG &DAG);
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
+    ///
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG);
 
     /// getTargetNodeName - This method returns the name of a target specific 
     //  DAG node.
@@ -112,7 +116,7 @@ namespace llvm {
               MVT VT) const;
   
     // Expand specifics
-    SDNode *ExpandADDSUB(SDNode *Op, SelectionDAG &DAG);
+    SDValue ExpandADDSUB(SDNode *Op, SelectionDAG &DAG);
   };
 }