diff --git a/plugins/woocommerce-blocks/assets/js/base/context/hooks/use-checkout-address.js b/plugins/woocommerce-blocks/assets/js/base/context/hooks/use-checkout-address.js
index 5f58b412a17..0ac8b09e573 100644
--- a/plugins/woocommerce-blocks/assets/js/base/context/hooks/use-checkout-address.js
+++ b/plugins/woocommerce-blocks/assets/js/base/context/hooks/use-checkout-address.js
@@ -87,10 +87,16 @@ export const useCheckoutAddress = () => {
 				previousBillingData.current = billingData;
 				setBillingData( shippingAddress );
 			} else {
+				const {
+					// We need to pluck out email and phone from previous billing data because they can be empty, causing the current email and phone to get emptied. See issue #4155
+					/* eslint-disable no-unused-vars */
+					email,
+					phone,
+					/* eslint-enable no-unused-vars */
+					...billingAddress
+				} = previousBillingData.current;
 				setBillingData( {
-					...previousBillingData.current,
-					email: undefined,
-					phone: undefined,
+					...billingAddress,
 				} );
 			}
 			currentShippingAsBilling.current = shippingAsBilling;